""" Scientific Gallery Generator This module generates static HTML galleries from scientific plot collections. It converts PDF plots to PNG thumbnails, creates responsive web interfaces, and organizes plots into hierarchical directory structures. Features: - PDF to PNG conversion with configurable DPI - Incremental updates (only converts when source is newer) - Jinja2 templating for consistent HTML generation - Support for nested folder structures - Responsive grid layout with search and navigation """ import subprocess import shutil from pathlib import Path from typing import Dict, Any, Optional from jinja2 import Environment, FileSystemLoader from orchestration.config import Config from orchestration.metadata import ( load_folder_metadata, merge_metadata, resolve_metadata_for_plot, save_metadata_cache ) CONFIG = Config.from_yaml("config.yaml") env = Environment(loader=FileSystemLoader(".")) template = env.get_template("templates/gallery.html") def convert_pdf_to_png(pdf_path: Path) -> None: """ Convert a PDF file to PNG format using ImageMagick. Only converts if the PNG doesn't exist or if the PDF is newer than the PNG (with a 30-second buffer to handle filesystem timing issues). Args: pdf_path: Path to the source PDF file Raises: subprocess.CalledProcessError: If ImageMagick conversion fails """ png_path = pdf_path.with_suffix(".png") if png_path.exists(): pdf_mtime = pdf_path.stat().st_mtime png_mtime = png_path.stat().st_mtime if png_mtime >= (pdf_mtime + 30): return else: print(f"PDF {pdf_path.name} is newer than PNG, reconverting...") print(f"Converting {pdf_path} → {png_path}") subprocess.run([ "convert", "-density", str(CONFIG.png_dpi), str(pdf_path), "-quality", "95", str(png_path) ], check=True) def needs_update(source_file: Path, target_file: Path) -> bool: """ Check if target file needs updating based on source modification time. Args: source_file: Path to the source file target_file: Path to the target file Returns: True if target needs update, False otherwise """ if not target_file.exists(): return True source_mtime = source_file.stat().st_mtime target_mtime = target_file.stat().st_mtime return source_mtime > (target_mtime + 30) def build_gallery(source_dir: Path, web_dir: Path, relative_path: Path = None, inherited_metadata: Optional[Dict[str, Any]] = None) -> None: """ Recursively build gallery structure from source directory. Processes all PDF files in the source directory, converts them to PNG, copies both to the web directory, and generates index.html files with navigation and thumbnails. Now includes metadata support. Args: source_dir: Source directory containing PDF files web_dir: Target web directory for gallery output relative_path: Relative path from gallery root (for navigation) inherited_metadata: Metadata inherited from parent directories """ if relative_path is None: relative_path = Path(".") if inherited_metadata is None: inherited_metadata = {} # Load folder-level metadata and merge with inherited metadata folder_metadata = load_folder_metadata(source_dir) current_metadata = merge_metadata(inherited_metadata, folder_metadata) pdf_files = list(source_dir.glob("*.pdf")) subdirs = [d for d in source_dir.iterdir() if d.is_dir()] items = [] plot_metadata_cache = {} for pdf_file in pdf_files: png_file = pdf_file.with_suffix(".png") web_pdf = web_dir / pdf_file.name web_png = web_dir / png_file.name if needs_update(pdf_file, web_pdf): print(f"Copying {pdf_file} to {web_pdf}") shutil.copy2(pdf_file, web_pdf) else: print(f"Skipping {pdf_file.name} (up to date)") if not png_file.exists(): convert_pdf_to_png(pdf_file) if needs_update(png_file, web_png): print(f"Copying {png_file} to {web_png}") shutil.copy2(png_file, web_png) else: print(f"Skipping {png_file.name} (up to date)") # Resolve metadata for this specific plot plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata) plot_metadata_cache[pdf_file.stem] = plot_metadata items.append({ "name": pdf_file.stem, "pdf_href": pdf_file.name, "png_href": png_file.name, "metadata": plot_metadata }) # Save metadata cache for this directory save_metadata_cache(web_dir, plot_metadata_cache) subdir_names = [] for subdir in subdirs: subdir_web = web_dir / subdir.name subdir_web.mkdir(exist_ok=True) subdir_relative = relative_path / subdir.name # Pass current metadata to subdirectories build_gallery(subdir, subdir_web, subdir_relative, current_metadata) subdir_names.append(subdir.name) output_html = web_dir / "index.html" if relative_path == Path("."): title = "Gallery" else: title = f"Gallery: {relative_path}" # Calculate statistics for current directory current_stats = calculate_directory_stats(web_dir) stats = { "file_count": len(items), "folder_count": len(subdir_names), "total_size": format_file_size(current_stats["total_size"]), "total_size_bytes": current_stats["total_size"] } # Calculate relative path to assets based on directory depth if relative_path == Path("."): assets_path = "../assets" else: # Count the number of directory levels to go back depth = len(relative_path.parts) assets_path = "../" * (depth + 1) + "assets" with output_html.open("w") as f: f.write(template.render( title=title, items=items, subdirs=subdir_names, relpath=str(relative_path), ui=CONFIG.ui, stats=stats, folder_metadata=current_metadata, assets_path=assets_path )) print(f"Generated {output_html}") def calculate_directory_stats(directory: Path) -> dict: """ Calculate statistics for a directory. Args: directory: Path to the directory to analyze Returns: Dictionary containing file count, folder count, and total size """ stats = { "file_count": 0, "folder_count": 0, "total_size": 0, "pdf_size": 0, "png_size": 0 } if not directory.exists(): return stats for item in directory.rglob("*"): if item.is_file(): stats["file_count"] += 1 size = item.stat().st_size stats["total_size"] += size if item.suffix.lower() == '.pdf': stats["pdf_size"] += size elif item.suffix.lower() == '.png': stats["png_size"] += size elif item.is_dir(): stats["folder_count"] += 1 return stats def format_file_size(size_bytes: int) -> str: """ Format file size in human readable format. Args: size_bytes: Size in bytes Returns: Formatted size string """ if size_bytes == 0: return "0 B" size_names = ["B", "KB", "MB", "GB", "TB"] size = float(size_bytes) i = 0 while size >= 1024 and i < len(size_names) - 1: size /= 1024 i += 1 return f"{size:.1f} {size_names[i]}" def main() -> None: """ Main entry point for gallery generation. Processes all configured sources and generates the complete gallery structure in the web directory. Cleans up the gallery directory on first run to ensure consistency. """ gallery_root = Path(CONFIG.web_folder) / CONFIG.plot_root if gallery_root.exists(): print(f"Gallery directory {gallery_root} exists, cleaning up...") shutil.rmtree(gallery_root) gallery_root.mkdir(parents=True, exist_ok=True) # Copy assets folder to the gallery root assets_src = Path("assets") assets_dst = gallery_root.parent / "assets" if assets_src.exists(): if assets_dst.exists(): shutil.rmtree(assets_dst) shutil.copytree(assets_src, assets_dst) print(f"Copied assets from {assets_src} to {assets_dst}") else: print(f"Warning: Assets directory {assets_src} not found") plot_metadata_cache = {} for source in CONFIG.sources: source_path = Path(source.path) if source_path.is_file() and source_path.suffix == '.pdf': source_web_dir = gallery_root / source.name source_web_dir.mkdir(parents=True, exist_ok=True) pdf_name = source_path.name png_name = source_path.with_suffix('.png').name web_pdf_path = source_web_dir / pdf_name web_png_path = source_web_dir / png_name source_png_path = source_path.with_suffix('.png') if needs_update(source_path, web_pdf_path): print(f"Copying {source_path} to {web_pdf_path}") shutil.copy2(source_path, web_pdf_path) else: print(f"Skipping {source_path.name} (up to date)") if not source_png_path.exists(): convert_pdf_to_png(source_path) if needs_update(source_png_path, web_png_path): print(f"Copying {source_png_path} to {web_png_path}") shutil.copy2(source_png_path, web_png_path) else: print(f"Skipping {source_png_path.name} (up to date)") # Resolve metadata for this single plot plot_metadata = resolve_metadata_for_plot(source_path, {}) plot_metadata_cache[source_path.stem] = plot_metadata items = [{ "name": source_path.stem, "pdf_href": pdf_name, "png_href": png_name, "metadata": plot_metadata }] # Save metadata cache for this directory plot_cache = {source_path.stem: plot_metadata} save_metadata_cache(source_web_dir, plot_cache) # Calculate statistics for single file current_stats = calculate_directory_stats(source_web_dir) stats = { "file_count": 1, "folder_count": 0, "total_size": format_file_size(current_stats["total_size"]), "total_size_bytes": current_stats["total_size"] } output_html = source_web_dir / "index.html" with output_html.open("w") as f: f.write(template.render( title=source.name, items=items, subdirs=[], relpath=source.name, ui=CONFIG.ui, stats=stats, folder_metadata={}, assets_path="../../assets" )) print(f"Generated {output_html}") print(f"Processed {source.name}: {source.path}") elif source_path.is_dir(): source_web_dir = gallery_root / source.name source_web_dir.mkdir(parents=True, exist_ok=True) build_gallery(source_path, source_web_dir, Path(source.name)) print(f"Processed {source.name}: {source.path}") else: print(f"Warning: Source {source.path} is neither a " f"directory nor a PDF file") print("Done") if __name__ == "__main__": main()