Fix missing top-level tree index.html. Clean up duplicates in main script

config.yaml

@@ -43,3 +43,5 @@ sources:
     path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/"
   - name: "needle_benchmarks"
     path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots"
+  - name: "aido_convergence_study"
+    path: "/work/kschmidt/aido/results_convergence/plots/"

generate_gallery.py

@@ -15,7 +15,6 @@ Features:
 
 import subprocess
 import shutil
-import sys
 from pathlib import Path
 from typing import Dict, Any, Optional
 from datetime import datetime
@@ -34,12 +33,12 @@ from utils.metadata import (
 CONFIG = Config.from_yaml("config.yaml")
 
 
-def datetime_from_timestamp(timestamp):
+def datetime_from_timestamp(timestamp: float) -> datetime:
     """Convert a Unix timestamp to a datetime object."""
     return datetime.fromtimestamp(timestamp)
 
 
-def strftime_filter(dt, fmt):
+def strftime_filter(dt: datetime, fmt: str) -> str:
     """Format a datetime object using strftime."""
     return dt.strftime(fmt)
 
@@ -50,6 +49,122 @@ env.filters['strftime'] = strftime_filter
 template = env.get_template("templates/gallery.html")
 
 
+def process_plot_files(
+        pdf_file: Path,
+        web_dir: Path,
+        current_metadata: Dict[str, Any] = None,
+) -> dict:
+    """
+    Process PDF and PNG files, handling conversion and copying.
+
+    Args:
+        pdf_file: Path to the source PDF file
+        web_dir: Target web directory
+        current_metadata: Current metadata dictionary for the plot
+
+    Returns:
+        Dictionary containing plot information
+    """
+    png_file = pdf_file.with_suffix(".png")
+    web_pdf = web_dir / pdf_file.name
+    web_png = web_dir / png_file.name
+
+    if needs_update(pdf_file, web_pdf):
+        shutil.copy2(pdf_file, web_pdf)
+    else:
+        print(f"Skipping {pdf_file.name} (up to date)")
+
+    if not png_file.exists():
+        convert_pdf_to_png(pdf_file)
+
+    if needs_update(png_file, web_png):
+        shutil.copy2(png_file, web_png)
+    else:
+        print(f"Skipping {png_file.name} (up to date)")
+
+    # Get source file creation time
+    source_creation_time = int(pdf_file.stat().st_ctime)
+
+    # Resolve metadata if provided
+    plot_metadata = {}
+    if current_metadata is not None:
+        plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
+
+    return {
+        "name": pdf_file.stem,
+        "pdf_href": pdf_file.name,
+        "png_href": png_file.name,
+        "metadata": plot_metadata,
+        "creation_time": source_creation_time
+    }
+
+
+def render_gallery_page(
+    web_dir: Path,
+    items: list,
+    subdirs: list,
+    relative_path: Path,
+    title: str = None,
+    metadata: dict = None
+) -> None:
+    """
+    Unified template rendering for all gallery pages.
+
+    Args:
+        web_dir: Target web directory
+        items: List of plot items
+        subdirs: List of subdirectory names
+        relative_path: Relative path from gallery root
+        title: Page title (optional)
+        metadata: Metadata dictionary (optional)
+    """
+    if title is None:
+        title = "Gallery" if relative_path == Path(
+            ".") else f"Gallery: {relative_path}"
+
+    if metadata is None:
+        metadata = {}
+
+    # Calculate statistics
+    current_stats = calculate_directory_stats(web_dir)
+    stats = {
+        "file_count": len(items),
+        "folder_count": len(subdirs),
+        "total_size": format_file_size(current_stats["total_size"]),
+        "total_size_bytes": current_stats["total_size"]
+    }
+
+    # Calculate relative path to assets
+    if relative_path == Path("."):
+        assets_path = "../assets"
+    else:
+        depth = len(relative_path.parts)
+        assets_path = "../" * (depth + 1) + "assets"
+
+    # For root level, show only directory structure
+    if relative_path == Path("."):
+        items = []
+
+    output_html = web_dir / "index.html"
+    with output_html.open("w") as f:
+        rendered_html = template.render(
+            title=title,
+            items=items,
+            subdirs=subdirs,
+            relpath=str(relative_path),
+            paths=CONFIG.paths,
+            ui=CONFIG.ui,
+            stats=stats,
+            folder_metadata=metadata,
+            assets_path=assets_path,
+            source_dir=str(web_dir),
+            metadata_file_path=get_metadata_file_path(web_dir)
+        )
+        f.write(rendered_html)
+
+    print(f"Generated {output_html}")
+
+
 def convert_pdf_to_png(pdf_path: Path) -> None:
     """
     Convert a PDF file to PNG format using ImageMagick.
@@ -73,7 +188,7 @@ def convert_pdf_to_png(pdf_path: Path) -> None:
         else:
             print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
 
-    print(f"Converting\n\t{pdf_path}\n  → {png_path}")
+    print(f"Converting {pdf_path} to PNG")
     subprocess.run([
         "convert",
         "-density", str(CONFIG.png_dpi),
@@ -103,9 +218,12 @@ def needs_update(source_file: Path, target_file: Path) -> bool:
     return source_mtime > (target_mtime + 30)
 
 
-def build_gallery(source_dir: Path, web_dir: Path,
+def build_gallery(
-                  relative_path: Path = None,
+    source_dir: Path,
-                  inherited_metadata: Optional[Dict[str, Any]] = None) -> None:
+    web_dir: Path,
+    relative_path: Path = None,
+    inherited_metadata: Optional[Dict[str, Any]] = None,
+) -> None:
     """
     Recursively build gallery structure from source directory.
 
@@ -125,117 +243,36 @@ def build_gallery(source_dir: Path, web_dir: Path,
     if inherited_metadata is None:
         inherited_metadata = {}
 
-    # Load folder-level metadata and merge with inherited metadata
     folder_metadata = load_folder_metadata(source_dir)
     current_metadata = merge_metadata(inherited_metadata, folder_metadata)
 
     pdf_files = list(source_dir.glob("*.pdf"))
-
-    subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
-
     items = []
     plot_metadata_cache = {}
 
     for pdf_file in pdf_files:
-        png_file = pdf_file.with_suffix(".png")
+        item = process_plot_files(pdf_file, web_dir, current_metadata)
+        items.append(item)
+        plot_metadata_cache[pdf_file.stem] = item["metadata"]
 
-        web_pdf = web_dir / pdf_file.name
-        web_png = web_dir / png_file.name
-
-        if needs_update(pdf_file, web_pdf):
-            print(f"Copying\n\t{pdf_file}\n  → {web_pdf}")
-            shutil.copy2(pdf_file, web_pdf)
-        else:
-            print(f"Skipping {pdf_file.name} (up to date)")
-
-        if not png_file.exists():
-            convert_pdf_to_png(pdf_file)
-
-        if needs_update(png_file, web_png):
-            print(f"Copying {png_file} to {web_png}")
-            shutil.copy2(png_file, web_png)
-        else:
-            print(f"Skipping {png_file.name} (up to date)")
-
-        # Resolve metadata for this specific plot
-        plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
-        plot_metadata_cache[pdf_file.stem] = plot_metadata
-
-        # Get source file creation time (in seconds since epoch)
-        source_creation_time = int(pdf_file.stat().st_ctime)
-
-        items.append({
-            "name": pdf_file.stem,
-            "pdf_href": pdf_file.name,
-            "png_href": png_file.name,
-            "metadata": plot_metadata,
-            "creation_time": source_creation_time
-        })
-
-    # Save metadata cache for this directory
     save_metadata_cache(web_dir, plot_metadata_cache)
 
+    subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
     subdir_names = []
     for subdir in subdirs:
         subdir_web = web_dir / subdir.name
         subdir_web.mkdir(exist_ok=True)
         subdir_relative = relative_path / subdir.name
-        # Pass current metadata to subdirectories
         build_gallery(subdir, subdir_web, subdir_relative, current_metadata)
         subdir_names.append(subdir.name)
 
-    output_html = web_dir / "index.html"
+    render_gallery_page(
-
+        web_dir=web_dir,
-    # Always regenerate HTML to ensure subdirectory changes are reflected
+        items=items,
-    # This ensures new subdirectories appear in navigation
+        subdirs=subdir_names,
-    force_regeneration = True
+        relative_path=relative_path,
-    if output_html.exists() and not force_regeneration:
+        metadata=current_metadata
-        html_mtime = output_html.stat().st_mtime
+    )
-        # Check if any subdirectory is newer than the HTML file
-        for subdir in subdirs:
-            if subdir.stat().st_mtime > html_mtime:
-                print(f"Subdirectory {subdir.name} is newer, forcing regeneration")
-                break
-
-    if relative_path == Path("."):
-        title = "Gallery"
-    else:
-        title = f"Gallery: {relative_path}"
-
-    # Calculate statistics for current directory
-    current_stats = calculate_directory_stats(web_dir)
-    stats = {
-        "file_count": len(items),
-        "folder_count": len(subdir_names),
-        "total_size": format_file_size(current_stats["total_size"]),
-        "total_size_bytes": current_stats["total_size"]
-    }
-
-    # Calculate relative path to assets based on directory depth
-    if relative_path == Path("."):
-        assets_path = "../assets"
-    else:
-        # Count directory levels to go back to gallery, then to public_html
-        depth = len(relative_path.parts)
-        assets_path = "../" * (depth + 1) + "assets"
-
-    with output_html.open("w") as f:
-        rendered_html = template.render(
-            title=title,
-            items=items,
-            subdirs=subdir_names,
-            relpath=str(relative_path),
-            paths=CONFIG.paths,
-            ui=CONFIG.ui,
-            stats=stats,
-            folder_metadata=current_metadata,
-            assets_path=assets_path,
-            source_dir=str(source_dir),
-            metadata_file_path=get_metadata_file_path(source_dir)
-        )
-        f.write(rendered_html)
-
-    print(f"Generated {output_html}")
 
 
 def calculate_directory_stats(directory: Path) -> dict:
@@ -253,7 +290,7 @@ def calculate_directory_stats(directory: Path) -> dict:
         "folder_count": 0,
         "total_size": 0,
         "pdf_size": 0,
-        "png_size": 0
+        "png_size": 0,
     }
 
     if not directory.exists():
@@ -298,21 +335,6 @@ def format_file_size(size_bytes: int) -> str:
     return f"{size:.1f} {size_names[i]}"
 
 
-def refresh_gallery_cgi():
-    """
-    CGI handler to refresh the gallery from a web request.
-    Outputs a minimal HTTP response and triggers gallery regeneration.
-    """
-    import traceback
-    print("Content-Type: text/plain\n")
-    try:
-        main()
-        print("Gallery refreshed successfully.")
-    except Exception as e:
-        print(f"Error refreshing gallery: {e}")
-        traceback.print_exc(file=sys.stdout)
-
-
 def main(clean_first: bool = False) -> None:
     """
     Main entry point for gallery generation.
@@ -329,15 +351,11 @@ def main(clean_first: bool = False) -> None:
         print(f"Cleaning gallery directory {gallery_root}...")
         shutil.rmtree(gallery_root)
 
-    # Ensure gallery root exists
     gallery_root.mkdir(parents=True, exist_ok=True)
-
-    # Always ensure assets are up to date
     assets_src = Path("assets")
     assets_dst = gallery_root.parent / "assets"
 
     if assets_src.exists():
-        # Update assets if they don't exist or are outdated
         main_css_src = assets_src / "css" / "main.css"
         main_css_dst = assets_dst / "css" / "main.css"
         if not assets_dst.exists() or needs_update(main_css_src, main_css_dst):
@@ -348,88 +366,39 @@ def main(clean_first: bool = False) -> None:
     else:
         print(f"Warning: Assets directory {assets_src} not found")
 
+    source_subdirs = []
+
     for source in CONFIG.sources:
         source_path = Path(source.path)
+        source_web_dir = gallery_root / source.name
+        source_web_dir.mkdir(parents=True, exist_ok=True)
+        source_subdirs.append(source.name)
 
         if source_path.is_file() and source_path.suffix == '.pdf':
-            source_web_dir = gallery_root / source.name
+            item = process_plot_files(source_path, source_web_dir)
-            source_web_dir.mkdir(parents=True, exist_ok=True)
+            render_gallery_page(
-
+                web_dir=source_web_dir,
-            pdf_name = source_path.name
+                items=[item],
-            png_name = source_path.with_suffix('.png').name
+                subdirs=[],
-
+                relative_path=Path(source.name)
-            web_pdf_path = source_web_dir / pdf_name
+            )
-            web_png_path = source_web_dir / png_name
-            source_png_path = source_path.with_suffix('.png')
-
-            if needs_update(source_path, web_pdf_path):
-                print(f"Copying {source_path} to {web_pdf_path}")
-                shutil.copy2(source_path, web_pdf_path)
-            else:
-                print(f"Skipping {source_path.name} (up to date)")
-
-            if not source_png_path.exists():
-                convert_pdf_to_png(source_path)
-
-            if needs_update(source_png_path, web_png_path):
-                print(f"Copying {source_png_path} to {web_png_path}")
-                shutil.copy2(source_png_path, web_png_path)
-            else:
-                print(f"Skipping {source_png_path.name} (up to date)")
-
-            # Get source file creation time for single file
-            source_creation_time = int(source_path.stat().st_ctime)
-
-            items = [{
-                "name": source_path.stem,
-                "pdf_href": pdf_name,
-                "png_href": png_name,
-                "creation_time": source_creation_time
-            }]
-
-            # Calculate statistics for single file
-            current_stats = calculate_directory_stats(source_web_dir)
-            stats = {
-                "file_count": 1,
-                "folder_count": 0,
-                "total_size": format_file_size(current_stats["total_size"]),
-                "total_size_bytes": current_stats["total_size"]
-            }
-
-            # Calculate relative path to assets for single file
-            # Single files are at depth 1 (gallery_root/source.name/index.html)
-            assets_path = "../assets"
-
-            output_html = source_web_dir / "index.html"
-            with output_html.open("w") as f:
-                f.write(template.render(
-                    title=source.name,
-                    items=items,
-                    subdirs=[],
-                    relpath=source.name,
-                    paths=CONFIG.paths,
-                    ui=CONFIG.ui,
-                    stats=stats,
-                    folder_metadata={},
-                    assets_path=assets_path,
-                    source_dir=str(source_path.parent),
-                    metadata_file_path=get_metadata_file_path(
-                        source_path.parent)
-                ))
-
-            print(f"Generated {output_html}")
-            print(f"Processed {source.name}: {source.path}")
-
         elif source_path.is_dir():
-            source_web_dir = gallery_root / source.name
-            source_web_dir.mkdir(parents=True, exist_ok=True)
             build_gallery(source_path, source_web_dir, Path(source.name))
-            print(f"Processed {source.name}: {source.path}")
         else:
-            print(f"Warning: Source {source.path} is neither a "
+            print(
-                  f"directory nor a PDF file")
+                f"Warning: Source {source.path} is neither a "
+                f"directory nor a PDF file. Skipping."
+            )
 
-    print("Done")
+        print(f"Processed {source.name}: {source.path}")
+
+    render_gallery_page(
+        web_dir=gallery_root,
+        items=[],
+        subdirs=source_subdirs,
+        relative_path=Path("."),
+        title="Gallery Root"
+    )
 
 
 if __name__ == "__main__":