Update and refactor

config.py

@@ -1,64 +1,128 @@
+"""
+Scientific Gallery Configuration Management
+
+This module provides dataclasses and utilities for managing configuration
+of the scientific gallery system, including paths, gallery settings,
+UI preferences, and data sources.
+"""
+
 from dataclasses import dataclass, field, asdict
 from pathlib import Path
-
 import yaml
 
 
+@dataclass
+class PathConfig:
+    """Configuration for system paths and directories."""
+    work_dir: str
+    web_folder: str
+    cgi_script: str
+    config_path: str
+
+
+@dataclass
+class GalleryConfig:
+    """Configuration for gallery generation and display settings."""
+    plot_root: str
+    png_dpi: int
+    backup_folder: str
+
+
+@dataclass
+class UIConfig:
+    """Configuration for user interface behavior and preferences."""
+    max_recent_plots: int
+    search_debounce_ms: int
+
+
 @dataclass
 class GalleryItem:
+    """Represents a single data source for the gallery."""
     name: str
     path: Path
 
 
 @dataclass
 class Config:
-    web_folder: str = ""
+    """
-    backup_folder: str = ""
+    Main configuration class that aggregates all gallery settings.
-    png_dpi: int = 400
+    
-    plot_root: str = "gallery"
+    Provides backward compatibility properties and methods for loading
+    configuration from YAML files.
+    """
+    paths: PathConfig
+    gallery: GalleryConfig
+    ui: UIConfig
     sources: list[GalleryItem] = field(default_factory=list)
 
+    @property
+    def web_folder(self):
+        """Backward compatibility property for web folder path."""
+        return self.paths.web_folder
+
+    @property
+    def png_dpi(self):
+        """Backward compatibility property for PNG conversion DPI."""
+        return self.gallery.png_dpi
+
+    @property
+    def plot_root(self):
+        """Backward compatibility property for plot root directory."""
+        return self.gallery.plot_root
+
+    @property
+    def backup_folder(self):
+        """Backward compatibility property for backup folder path."""
+        return self.gallery.backup_folder
+
     @classmethod
     def from_yaml(cls, yaml_file: str) -> "Config":
         """
         Load configuration from a YAML file.
 
         Args:
-            yaml_file (str): Path to the YAML file.
+            yaml_file: Path to the YAML configuration file
-            strict (bool):
-                If True, raises an error if a key in the YAML file does not exist in the Config class.
-                If False (default), adds all keys as attributes
 
         Returns:
-            Config: Instance of this class
+            Config instance with loaded settings
+
+        Raises:
+            FileNotFoundError: If the YAML file doesn't exist
+            yaml.YAMLError: If the YAML file is malformed
         """
-
         with open(yaml_file, "r") as f:
-            new_config: dict = yaml.safe_load(f)
+            data = yaml.safe_load(f)
 
-        new_config["sources"] = [
+        paths_data = data.get('paths', {})
-            GalleryItem(name=src["name"], path=Path(src["path"]))
+        gallery_data = data.get('gallery', {})
-            for src in new_config.get("sources", False)
+        ui_data = data.get('ui', {})
+        sources_data = data.get('sources', [])
+
+        paths = PathConfig(**paths_data)
+        gallery = GalleryConfig(**gallery_data)
+        ui = UIConfig(**ui_data)
+
+        sources = [
+            GalleryItem(name=source["name"], path=Path(source["path"]))
+            for source in sources_data
         ]
 
-        instance = cls(**{
+        return cls(paths=paths, gallery=gallery, ui=ui, sources=sources)
-            k: v
-            for k, v in new_config.items()
-        })
-
-        return instance
 
     def to_yaml(self, yaml_file: str) -> None:
         """
         Save the current configuration to a YAML file.
 
         Args:
-            yaml_file (str): Path to the YAML file.
+            yaml_file: Path where to save the YAML configuration
+
+        Raises:
+            IOError: If unable to write to the specified file
         """
         with open(yaml_file, "w") as f:
             yaml.dump(asdict(self), f, default_flow_style=False)
 
 
 if __name__ == "__main__":
-    config = Config()
+    config = Config.from_yaml("config.yaml")
-    config.to_yaml("config.yaml")
+    print("Loaded config successfully:", config)

config.yaml

@@ -1,7 +1,41 @@
-backup_folder: ''
+# Gallery Configuration
-plot_root: gallery
+# ===================
-png_dpi: 400
+
+# Paths Configuration
+paths:
+  # Working directory where the script runs from
+  work_dir: "/work/kschmidt/web"
+  
+  # Web hosting directory where gallery files are served
+  web_folder: "/web/kschmidt/public_html/"
+  
+  # CGI script path (relative to web folder)
+  cgi_script: "cgi-bin/refresh_gallery.py"
+  
+  # Config file path for CGI scripts
+  config_path: "/work/kschmidt/web"
+
+# Gallery Settings
+gallery:
+  # Root folder name for plots in web directory
+  plot_root: "gallery"
+  
+  # PNG conversion quality
+  png_dpi: 400
+  
+  # Backup folder (leave empty to disable)
+  backup_folder: ""
+
+# UI Settings
+ui:
+  # Maximum number of recent plots to track
+  max_recent_plots: 20
+  
+  # Search settings
+  search_debounce_ms: 300
+
+# Data Sources
+# Each source represents a collection of plots to include in the gallery
 sources: 
   - name: "ttbar_analysis"
     path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/"
-web_folder: '/web/kschmidt/public_html/'

generate_gallery.py

@@ -1,6 +1,20 @@
+"""
+Scientific Gallery Generator
+
+This module generates static HTML galleries from scientific plot collections.
+It converts PDF plots to PNG thumbnails, creates responsive web interfaces,
+and organizes plots into hierarchical directory structures.
+
+Features:
+- PDF to PNG conversion with configurable DPI
+- Incremental updates (only converts when source is newer)
+- Jinja2 templating for consistent HTML generation
+- Support for nested folder structures
+- Responsive grid layout with search and navigation
+"""
+
 import subprocess
 import shutil
-
 from pathlib import Path
 from jinja2 import Environment, FileSystemLoader
 from config import Config
@@ -12,15 +26,25 @@ env = Environment(loader=FileSystemLoader("."))
 template = env.get_template("template.html")
 
 
-def convert_pdf_to_png(pdf_path: Path):
+def convert_pdf_to_png(pdf_path: Path) -> None:
+    """
+    Convert a PDF file to PNG format using ImageMagick.
+
+    Only converts if the PNG doesn't exist or if the PDF is newer than
+    the PNG (with a 30-second buffer to handle filesystem timing issues).
+
+    Args:
+        pdf_path: Path to the source PDF file
+
+    Raises:
+        subprocess.CalledProcessError: If ImageMagick conversion fails
+    """
     png_path = pdf_path.with_suffix(".png")
 
-    # Check if PNG exists and is newer than PDF (with 30 second buffer)
     if png_path.exists():
         pdf_mtime = pdf_path.stat().st_mtime
         png_mtime = png_path.stat().st_mtime
-        if png_mtime >= (pdf_mtime + 30):  # 30 second buffer
+        if png_mtime >= (pdf_mtime + 30):
-            # PNG is up to date, no need to convert
             return
         else:
             print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
@@ -36,72 +60,85 @@ def convert_pdf_to_png(pdf_path: Path):
 
 
 def needs_update(source_file: Path, target_file: Path) -> bool:
-    """Check if target file needs to be updated based on source file modification time"""
+    """
+    Check if target file needs updating based on source modification time.
+
+    Args:
+        source_file: Path to the source file
+        target_file: Path to the target file
+
+    Returns:
+        True if target needs update, False otherwise
+    """
     if not target_file.exists():
         return True
 
     source_mtime = source_file.stat().st_mtime
     target_mtime = target_file.stat().st_mtime
 
-    # Return True if source is newer than target (with 30 second buffer)
     return source_mtime > (target_mtime + 30)
 
 
-def build_gallery(source_dir: Path, web_dir: Path, relative_path: Path = None):
+def build_gallery(source_dir: Path, web_dir: Path,
+                  relative_path: Path = None) -> None:
     """
-    Build gallery for a directory, copying files to web directory and generating index.html
+    Recursively build gallery structure from source directory.
+
+    Processes all PDF files in the source directory, converts them to PNG,
+    copies both to the web directory, and generates index.html files with
+    navigation and thumbnails.
 
     Args:
-        source_dir: Source directory containing PDFs
+        source_dir: Source directory containing PDF files
-        web_dir: Target web directory
+        web_dir: Target web directory for gallery output
-        relative_path: Relative path for navigation (None for root)
+        relative_path: Relative path from gallery root (for navigation)
     """
     if relative_path is None:
         relative_path = Path(".")
 
-    # Ensure web directory exists
+    pdf_files = list(source_dir.glob("*.pdf"))
-    target_dir = web_dir / relative_path
+    subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
-    target_dir.mkdir(parents=True, exist_ok=True)
 
-    # Find all PDFs in source directory
-    pdfs = sorted(source_dir.glob("*.pdf"))
     items = []
 
-    for pdf in pdfs:
+    for pdf_file in pdf_files:
-        # Convert PDF to PNG in source directory
+        png_file = pdf_file.with_suffix(".png")
-        convert_pdf_to_png(pdf)
-        png_path = pdf.with_suffix(".png")
 
-        # Copy both PDF and PNG to target directory only if needed
+        web_pdf = web_dir / pdf_file.name
-        target_pdf = target_dir / pdf.name
+        web_png = web_dir / png_file.name
-        target_png = target_dir / png_path.name
 
-        if needs_update(pdf, target_pdf):
+        if needs_update(pdf_file, web_pdf):
-            print(f"Copying {pdf.name} to {target_pdf}")
+            print(f"Copying {pdf_file} to {web_pdf}")
-            shutil.copy2(pdf, target_pdf)
+            shutil.copy2(pdf_file, web_pdf)
         else:
-            print(f"Skipping {pdf.name} (up to date)")
+            print(f"Skipping {pdf_file.name} (up to date)")
 
-        if png_path.exists() and needs_update(png_path, target_png):
+        if not png_file.exists():
-            print(f"Copying {png_path.name} to {target_png}")
+            convert_pdf_to_png(pdf_file)
-            shutil.copy2(png_path, target_png)
+
-        elif png_path.exists():
+        if needs_update(png_file, web_png):
-            print(f"Skipping {png_path.name} (up to date)")
+            print(f"Copying {png_file} to {web_png}")
+            shutil.copy2(png_file, web_png)
+        else:
+            print(f"Skipping {png_file.name} (up to date)")
 
         items.append({
-            "name": pdf.name,
+            "name": pdf_file.stem,
-            "pdf_href": pdf.name,
+            "pdf_href": pdf_file.name,
-            "png_href": png_path.name
+            "png_href": png_file.name
         })
 
-    # Find subdirectories in source
+    subdir_names = []
-    subdirs = sorted([d.name for d in source_dir.iterdir() if d.is_dir()])
+    for subdir in subdirs:
+        subdir_web = web_dir / subdir.name
+        subdir_web.mkdir(exist_ok=True)
+        subdir_relative = relative_path / subdir.name
+        build_gallery(subdir, subdir_web, subdir_relative)
+        subdir_names.append(subdir.name)
 
-    # Generate index.html in target directory
+    output_html = web_dir / "index.html"
-    output_html = target_dir / "index.html"
 
-    # Create a meaningful title
+    if relative_path == Path("."):
-    if str(relative_path) == ".":
         title = "Gallery"
     else:
         title = f"Gallery: {relative_path}"
@@ -110,75 +147,89 @@ def build_gallery(source_dir: Path, web_dir: Path, relative_path: Path = None):
         f.write(template.render(
             title=title,
             items=items,
-            subdirs=subdirs,
+            subdirs=subdir_names,
-            relpath=str(relative_path)
+            relpath=str(relative_path),
+            ui=CONFIG.ui
         ))
 
     print(f"Generated {output_html}")
 
-    # Recursively process subdirectories
-    for subdir in subdirs:
-        source_subdir = source_dir / subdir
-        new_relative_path = relative_path / subdir
-        build_gallery(source_subdir, web_dir, new_relative_path)
 
+def main() -> None:
+    """
+    Main entry point for gallery generation.
 
-def loop_over_sources():
+    Processes all configured sources and generates the complete gallery
-    web_root = Path(CONFIG.web_folder)
+    structure in the web directory. Cleans up the gallery directory
+    on first run to ensure consistency.
+    """
+    gallery_root = Path(CONFIG.web_folder) / CONFIG.plot_root
+
+    if gallery_root.exists():
+        print(f"Gallery directory {gallery_root} exists, cleaning up...")
+        shutil.rmtree(gallery_root)
+
+    gallery_root.mkdir(parents=True, exist_ok=True)
 
     for source in CONFIG.sources:
-        print(f"Processing {source.name}: {source.path}")
+        source_path = Path(source.path)
-        if source.path.is_dir():
-            # Create a subdirectory based on plot_root and source name
-            source_web_dir = web_root / CONFIG.plot_root / source.name
-            build_gallery(source.path, source_web_dir)
-            print(f"Processed {source.name}: {source.path}")
-        elif source.path.suffix == ".pdf":
-            # For single PDF files, copy to web root with plot_root and source name as directory
-            source_web_dir = web_root / CONFIG.plot_root / source.name
-            convert_pdf_to_png(source.path)
-            png_path = source.path.with_suffix(".png")
 
-            # Ensure web directory exists
+        if source_path.is_file() and source_path.suffix == '.pdf':
+            source_web_dir = gallery_root / source.name
             source_web_dir.mkdir(parents=True, exist_ok=True)
 
-            # Copy PDF and PNG to web directory only if needed
+            pdf_name = source_path.name
-            target_pdf = source_web_dir / source.path.name
+            png_name = source_path.with_suffix('.png').name
-            target_png = source_web_dir / png_path.name
 
-            if needs_update(source.path, target_pdf):
+            web_pdf_path = source_web_dir / pdf_name
-                print(f"Copying {source.path.name} to {target_pdf}")
+            web_png_path = source_web_dir / png_name
-                shutil.copy2(source.path, target_pdf)
+            source_png_path = source_path.with_suffix('.png')
+
+            if needs_update(source_path, web_pdf_path):
+                print(f"Copying {source_path} to {web_pdf_path}")
+                shutil.copy2(source_path, web_pdf_path)
             else:
-                print(f"Skipping {source.path.name} (up to date)")
+                print(f"Skipping {source_path.name} (up to date)")
 
-            if png_path.exists() and needs_update(png_path, target_png):
+            if not source_png_path.exists():
-                print(f"Copying {png_path.name} to {target_png}")
+                convert_pdf_to_png(source_path)
-                shutil.copy2(png_path, target_png)
+
-            elif png_path.exists():
+            if needs_update(source_png_path, web_png_path):
-                print(f"Skipping {png_path.name} (up to date)")
+                print(f"Copying {source_png_path} to {web_png_path}")
+                shutil.copy2(source_png_path, web_png_path)
+            else:
+                print(f"Skipping {source_png_path.name} (up to date)")
 
-            # Create items list for template
             items = [{
-                "name": source.path.name,
+                "name": source_path.stem,
-                "pdf_href": source.path.name,
+                "pdf_href": pdf_name,
-                "png_href": png_path.name
+                "png_href": png_name
             }]
 
-            # Generate index.html for single PDF
             output_html = source_web_dir / "index.html"
             with output_html.open("w") as f:
                 f.write(template.render(
                     title=source.name,
                     items=items,
                     subdirs=[],
-                    relpath=source.name
+                    relpath=source.name,
+                    ui=CONFIG.ui
                 ))
 
             print(f"Generated {output_html}")
             print(f"Processed {source.name}: {source.path}")
 
+        elif source_path.is_dir():
+            source_web_dir = gallery_root / source.name
+            source_web_dir.mkdir(parents=True, exist_ok=True)
+            build_gallery(source_path, source_web_dir, Path(source.name))
+            print(f"Processed {source.name}: {source.path}")
+        else:
+            print(f"Warning: Source {source.path} is neither a "
+                  f"directory nor a PDF file")
+
+    print("Done")
+
 
 if __name__ == "__main__":
-    loop_over_sources()
+    main()
-    print("Done")