From 041c761df0394e2f5793fa3968d8dab304d8e32e Mon Sep 17 00:00:00 2001 From: Kylian Schmidt Date: Fri, 26 Sep 2025 08:53:37 +0200 Subject: [PATCH] Fix missing top-level tree index.html. Clean up duplicates in main script --- config.yaml | 2 + generate_gallery.py | 355 ++++++++++++++++++++------------------------ 2 files changed, 164 insertions(+), 193 deletions(-) diff --git a/config.yaml b/config.yaml index 67f08af..bb35d8c 100644 --- a/config.yaml +++ b/config.yaml @@ -43,3 +43,5 @@ sources: path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/" - name: "needle_benchmarks" path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots" + - name: "aido_convergence_study" + path: "/work/kschmidt/aido/results_convergence/plots/" diff --git a/generate_gallery.py b/generate_gallery.py index 3987ad6..7b148ff 100644 --- a/generate_gallery.py +++ b/generate_gallery.py @@ -15,7 +15,6 @@ Features: import subprocess import shutil -import sys from pathlib import Path from typing import Dict, Any, Optional from datetime import datetime @@ -34,12 +33,12 @@ from utils.metadata import ( CONFIG = Config.from_yaml("config.yaml") -def datetime_from_timestamp(timestamp): +def datetime_from_timestamp(timestamp: float) -> datetime: """Convert a Unix timestamp to a datetime object.""" return datetime.fromtimestamp(timestamp) -def strftime_filter(dt, fmt): +def strftime_filter(dt: datetime, fmt: str) -> str: """Format a datetime object using strftime.""" return dt.strftime(fmt) @@ -50,6 +49,122 @@ env.filters['strftime'] = strftime_filter template = env.get_template("templates/gallery.html") +def process_plot_files( + pdf_file: Path, + web_dir: Path, + current_metadata: Dict[str, Any] = None, +) -> dict: + """ + Process PDF and PNG files, handling conversion and copying. + + Args: + pdf_file: Path to the source PDF file + web_dir: Target web directory + current_metadata: Current metadata dictionary for the plot + + Returns: + Dictionary containing plot information + """ + png_file = pdf_file.with_suffix(".png") + web_pdf = web_dir / pdf_file.name + web_png = web_dir / png_file.name + + if needs_update(pdf_file, web_pdf): + shutil.copy2(pdf_file, web_pdf) + else: + print(f"Skipping {pdf_file.name} (up to date)") + + if not png_file.exists(): + convert_pdf_to_png(pdf_file) + + if needs_update(png_file, web_png): + shutil.copy2(png_file, web_png) + else: + print(f"Skipping {png_file.name} (up to date)") + + # Get source file creation time + source_creation_time = int(pdf_file.stat().st_ctime) + + # Resolve metadata if provided + plot_metadata = {} + if current_metadata is not None: + plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata) + + return { + "name": pdf_file.stem, + "pdf_href": pdf_file.name, + "png_href": png_file.name, + "metadata": plot_metadata, + "creation_time": source_creation_time + } + + +def render_gallery_page( + web_dir: Path, + items: list, + subdirs: list, + relative_path: Path, + title: str = None, + metadata: dict = None +) -> None: + """ + Unified template rendering for all gallery pages. + + Args: + web_dir: Target web directory + items: List of plot items + subdirs: List of subdirectory names + relative_path: Relative path from gallery root + title: Page title (optional) + metadata: Metadata dictionary (optional) + """ + if title is None: + title = "Gallery" if relative_path == Path( + ".") else f"Gallery: {relative_path}" + + if metadata is None: + metadata = {} + + # Calculate statistics + current_stats = calculate_directory_stats(web_dir) + stats = { + "file_count": len(items), + "folder_count": len(subdirs), + "total_size": format_file_size(current_stats["total_size"]), + "total_size_bytes": current_stats["total_size"] + } + + # Calculate relative path to assets + if relative_path == Path("."): + assets_path = "../assets" + else: + depth = len(relative_path.parts) + assets_path = "../" * (depth + 1) + "assets" + + # For root level, show only directory structure + if relative_path == Path("."): + items = [] + + output_html = web_dir / "index.html" + with output_html.open("w") as f: + rendered_html = template.render( + title=title, + items=items, + subdirs=subdirs, + relpath=str(relative_path), + paths=CONFIG.paths, + ui=CONFIG.ui, + stats=stats, + folder_metadata=metadata, + assets_path=assets_path, + source_dir=str(web_dir), + metadata_file_path=get_metadata_file_path(web_dir) + ) + f.write(rendered_html) + + print(f"Generated {output_html}") + + def convert_pdf_to_png(pdf_path: Path) -> None: """ Convert a PDF file to PNG format using ImageMagick. @@ -73,7 +188,7 @@ def convert_pdf_to_png(pdf_path: Path) -> None: else: print(f"PDF {pdf_path.name} is newer than PNG, reconverting...") - print(f"Converting\n\t{pdf_path}\n → {png_path}") + print(f"Converting {pdf_path} to PNG") subprocess.run([ "convert", "-density", str(CONFIG.png_dpi), @@ -103,9 +218,12 @@ def needs_update(source_file: Path, target_file: Path) -> bool: return source_mtime > (target_mtime + 30) -def build_gallery(source_dir: Path, web_dir: Path, - relative_path: Path = None, - inherited_metadata: Optional[Dict[str, Any]] = None) -> None: +def build_gallery( + source_dir: Path, + web_dir: Path, + relative_path: Path = None, + inherited_metadata: Optional[Dict[str, Any]] = None, +) -> None: """ Recursively build gallery structure from source directory. @@ -125,117 +243,36 @@ def build_gallery(source_dir: Path, web_dir: Path, if inherited_metadata is None: inherited_metadata = {} - # Load folder-level metadata and merge with inherited metadata folder_metadata = load_folder_metadata(source_dir) current_metadata = merge_metadata(inherited_metadata, folder_metadata) pdf_files = list(source_dir.glob("*.pdf")) - - subdirs = [d for d in source_dir.iterdir() if d.is_dir()] - items = [] plot_metadata_cache = {} for pdf_file in pdf_files: - png_file = pdf_file.with_suffix(".png") + item = process_plot_files(pdf_file, web_dir, current_metadata) + items.append(item) + plot_metadata_cache[pdf_file.stem] = item["metadata"] - web_pdf = web_dir / pdf_file.name - web_png = web_dir / png_file.name - - if needs_update(pdf_file, web_pdf): - print(f"Copying\n\t{pdf_file}\n → {web_pdf}") - shutil.copy2(pdf_file, web_pdf) - else: - print(f"Skipping {pdf_file.name} (up to date)") - - if not png_file.exists(): - convert_pdf_to_png(pdf_file) - - if needs_update(png_file, web_png): - print(f"Copying {png_file} to {web_png}") - shutil.copy2(png_file, web_png) - else: - print(f"Skipping {png_file.name} (up to date)") - - # Resolve metadata for this specific plot - plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata) - plot_metadata_cache[pdf_file.stem] = plot_metadata - - # Get source file creation time (in seconds since epoch) - source_creation_time = int(pdf_file.stat().st_ctime) - - items.append({ - "name": pdf_file.stem, - "pdf_href": pdf_file.name, - "png_href": png_file.name, - "metadata": plot_metadata, - "creation_time": source_creation_time - }) - - # Save metadata cache for this directory save_metadata_cache(web_dir, plot_metadata_cache) + subdirs = [d for d in source_dir.iterdir() if d.is_dir()] subdir_names = [] for subdir in subdirs: subdir_web = web_dir / subdir.name subdir_web.mkdir(exist_ok=True) subdir_relative = relative_path / subdir.name - # Pass current metadata to subdirectories build_gallery(subdir, subdir_web, subdir_relative, current_metadata) subdir_names.append(subdir.name) - output_html = web_dir / "index.html" - - # Always regenerate HTML to ensure subdirectory changes are reflected - # This ensures new subdirectories appear in navigation - force_regeneration = True - if output_html.exists() and not force_regeneration: - html_mtime = output_html.stat().st_mtime - # Check if any subdirectory is newer than the HTML file - for subdir in subdirs: - if subdir.stat().st_mtime > html_mtime: - print(f"Subdirectory {subdir.name} is newer, forcing regeneration") - break - - if relative_path == Path("."): - title = "Gallery" - else: - title = f"Gallery: {relative_path}" - - # Calculate statistics for current directory - current_stats = calculate_directory_stats(web_dir) - stats = { - "file_count": len(items), - "folder_count": len(subdir_names), - "total_size": format_file_size(current_stats["total_size"]), - "total_size_bytes": current_stats["total_size"] - } - - # Calculate relative path to assets based on directory depth - if relative_path == Path("."): - assets_path = "../assets" - else: - # Count directory levels to go back to gallery, then to public_html - depth = len(relative_path.parts) - assets_path = "../" * (depth + 1) + "assets" - - with output_html.open("w") as f: - rendered_html = template.render( - title=title, - items=items, - subdirs=subdir_names, - relpath=str(relative_path), - paths=CONFIG.paths, - ui=CONFIG.ui, - stats=stats, - folder_metadata=current_metadata, - assets_path=assets_path, - source_dir=str(source_dir), - metadata_file_path=get_metadata_file_path(source_dir) - ) - f.write(rendered_html) - - print(f"Generated {output_html}") + render_gallery_page( + web_dir=web_dir, + items=items, + subdirs=subdir_names, + relative_path=relative_path, + metadata=current_metadata + ) def calculate_directory_stats(directory: Path) -> dict: @@ -253,7 +290,7 @@ def calculate_directory_stats(directory: Path) -> dict: "folder_count": 0, "total_size": 0, "pdf_size": 0, - "png_size": 0 + "png_size": 0, } if not directory.exists(): @@ -298,21 +335,6 @@ def format_file_size(size_bytes: int) -> str: return f"{size:.1f} {size_names[i]}" -def refresh_gallery_cgi(): - """ - CGI handler to refresh the gallery from a web request. - Outputs a minimal HTTP response and triggers gallery regeneration. - """ - import traceback - print("Content-Type: text/plain\n") - try: - main() - print("Gallery refreshed successfully.") - except Exception as e: - print(f"Error refreshing gallery: {e}") - traceback.print_exc(file=sys.stdout) - - def main(clean_first: bool = False) -> None: """ Main entry point for gallery generation. @@ -329,15 +351,11 @@ def main(clean_first: bool = False) -> None: print(f"Cleaning gallery directory {gallery_root}...") shutil.rmtree(gallery_root) - # Ensure gallery root exists gallery_root.mkdir(parents=True, exist_ok=True) - - # Always ensure assets are up to date assets_src = Path("assets") assets_dst = gallery_root.parent / "assets" if assets_src.exists(): - # Update assets if they don't exist or are outdated main_css_src = assets_src / "css" / "main.css" main_css_dst = assets_dst / "css" / "main.css" if not assets_dst.exists() or needs_update(main_css_src, main_css_dst): @@ -348,88 +366,39 @@ def main(clean_first: bool = False) -> None: else: print(f"Warning: Assets directory {assets_src} not found") + source_subdirs = [] + for source in CONFIG.sources: source_path = Path(source.path) + source_web_dir = gallery_root / source.name + source_web_dir.mkdir(parents=True, exist_ok=True) + source_subdirs.append(source.name) if source_path.is_file() and source_path.suffix == '.pdf': - source_web_dir = gallery_root / source.name - source_web_dir.mkdir(parents=True, exist_ok=True) - - pdf_name = source_path.name - png_name = source_path.with_suffix('.png').name - - web_pdf_path = source_web_dir / pdf_name - web_png_path = source_web_dir / png_name - source_png_path = source_path.with_suffix('.png') - - if needs_update(source_path, web_pdf_path): - print(f"Copying {source_path} to {web_pdf_path}") - shutil.copy2(source_path, web_pdf_path) - else: - print(f"Skipping {source_path.name} (up to date)") - - if not source_png_path.exists(): - convert_pdf_to_png(source_path) - - if needs_update(source_png_path, web_png_path): - print(f"Copying {source_png_path} to {web_png_path}") - shutil.copy2(source_png_path, web_png_path) - else: - print(f"Skipping {source_png_path.name} (up to date)") - - # Get source file creation time for single file - source_creation_time = int(source_path.stat().st_ctime) - - items = [{ - "name": source_path.stem, - "pdf_href": pdf_name, - "png_href": png_name, - "creation_time": source_creation_time - }] - - # Calculate statistics for single file - current_stats = calculate_directory_stats(source_web_dir) - stats = { - "file_count": 1, - "folder_count": 0, - "total_size": format_file_size(current_stats["total_size"]), - "total_size_bytes": current_stats["total_size"] - } - - # Calculate relative path to assets for single file - # Single files are at depth 1 (gallery_root/source.name/index.html) - assets_path = "../assets" - - output_html = source_web_dir / "index.html" - with output_html.open("w") as f: - f.write(template.render( - title=source.name, - items=items, - subdirs=[], - relpath=source.name, - paths=CONFIG.paths, - ui=CONFIG.ui, - stats=stats, - folder_metadata={}, - assets_path=assets_path, - source_dir=str(source_path.parent), - metadata_file_path=get_metadata_file_path( - source_path.parent) - )) - - print(f"Generated {output_html}") - print(f"Processed {source.name}: {source.path}") - + item = process_plot_files(source_path, source_web_dir) + render_gallery_page( + web_dir=source_web_dir, + items=[item], + subdirs=[], + relative_path=Path(source.name) + ) elif source_path.is_dir(): - source_web_dir = gallery_root / source.name - source_web_dir.mkdir(parents=True, exist_ok=True) build_gallery(source_path, source_web_dir, Path(source.name)) - print(f"Processed {source.name}: {source.path}") else: - print(f"Warning: Source {source.path} is neither a " - f"directory nor a PDF file") + print( + f"Warning: Source {source.path} is neither a " + f"directory nor a PDF file. Skipping." + ) - print("Done") + print(f"Processed {source.name}: {source.path}") + + render_gallery_page( + web_dir=gallery_root, + items=[], + subdirs=source_subdirs, + relative_path=Path("."), + title="Gallery Root" + ) if __name__ == "__main__":