diff --git a/config.yaml b/config.yaml index bb35d8c..3e54157 100644 --- a/config.yaml +++ b/config.yaml @@ -43,5 +43,7 @@ sources: path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/" - name: "needle_benchmarks" path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots" + - name: "needle_fair_universe" + path: "/ceph/kschmidt/needle/plots/fair_universe/" - name: "aido_convergence_study" path: "/work/kschmidt/aido/results_convergence/plots/" diff --git a/generate_gallery.py b/generate_gallery.py index 527b073..9423e89 100644 --- a/generate_gallery.py +++ b/generate_gallery.py @@ -13,7 +13,6 @@ Features: - Responsive grid layout with search and navigation """ -import subprocess import shutil from pathlib import Path from typing import Dict, Any, Optional @@ -24,9 +23,12 @@ from utils.config import Config from utils.metadata import ( load_folder_metadata, merge_metadata, - resolve_metadata_for_plot, save_metadata_cache, - get_metadata_file_path +) +from utils.processing import ( + process_plot_files, + needs_update, + render_gallery_page, ) @@ -49,213 +51,6 @@ env.filters['strftime'] = strftime_filter template = env.get_template("templates/gallery.html") -def process_html_file(html_file: Path, web_dir: Path, current_metadata: Dict[str, Any] = None) -> dict: - """ - Process HTML plot file, copying it to web directory. - - Args: - html_file: Path to the source HTML file - web_dir: Target web directory - current_metadata: Current metadata dictionary for the plot - - Returns: - Dictionary containing plot information - """ - web_html = web_dir / html_file.name - - if needs_update(html_file, web_html): - shutil.copy2(html_file, web_html) - else: - print(f"Skipping {html_file.name} (up to date)") - - # Get source file creation time - source_creation_time = int(html_file.stat().st_ctime) - - # Resolve metadata if provided - plot_metadata = {} - if current_metadata is not None: - plot_metadata = resolve_metadata_for_plot(html_file, current_metadata) - - return { - "name": html_file.stem, - "html_href": html_file.name, - "is_html": True, - "metadata": plot_metadata, - "creation_time": source_creation_time - } - -def process_plot_files( - plot_file: Path, - web_dir: Path, - current_metadata: Dict[str, Any] = None, -) -> dict: - """ - Process plot files (PDF/PNG or HTML), handling conversion and copying. - - Args: - plot_file: Path to the source plot file (PDF or HTML) - web_dir: Target web directory - current_metadata: Current metadata dictionary for the plot - - Returns: - Dictionary containing plot information - """ - if plot_file.suffix.lower() == '.html': - return process_html_file(plot_file, web_dir, current_metadata) - - # Handle PDF files - png_file = plot_file.with_suffix(".png") - web_pdf = web_dir / plot_file.name - web_png = web_dir / png_file.name - - if needs_update(plot_file, web_pdf): - shutil.copy2(plot_file, web_pdf) - else: - print(f"Skipping {plot_file.name} (up to date)") - - if not png_file.exists(): - convert_pdf_to_png(plot_file) - - if needs_update(png_file, web_png): - shutil.copy2(png_file, web_png) - else: - print(f"Skipping {png_file.name} (up to date)") - - source_creation_time = int(plot_file.stat().st_ctime) - - plot_metadata = {} - if current_metadata is not None: - plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata) - - return { - "name": plot_file.stem, - "pdf_href": plot_file.name, - "png_href": png_file.name, - "is_html": False, - "metadata": plot_metadata, - "creation_time": source_creation_time - } - - -def render_gallery_page( - web_dir: Path, - items: list, - subdirs: list, - relative_path: Path, - title: str = None, - metadata: dict = None -) -> None: - """ - Unified template rendering for all gallery pages. - - Args: - web_dir: Target web directory - items: List of plot items - subdirs: List of subdirectory names - relative_path: Relative path from gallery root - title: Page title (optional) - metadata: Metadata dictionary (optional) - """ - if title is None: - title = "Gallery" if relative_path == Path( - ".") else f"Gallery: {relative_path}" - - if metadata is None: - metadata = {} - - # Calculate statistics - current_stats = calculate_directory_stats(web_dir) - stats = { - "file_count": len(items), - "folder_count": len(subdirs), - "total_size": format_file_size(current_stats["total_size"]), - "total_size_bytes": current_stats["total_size"] - } - - # Calculate relative path to assets - if relative_path == Path("."): - assets_path = "../assets" - else: - depth = len(relative_path.parts) - assets_path = "../" * (depth + 1) + "assets" - - # For root level, show only directory structure - if relative_path == Path("."): - items = [] - - output_html = web_dir / "index.html" - with output_html.open("w") as f: - rendered_html = template.render( - title=title, - items=items, - subdirs=subdirs, - relpath=str(relative_path), - paths=CONFIG.paths, - ui=CONFIG.ui, - stats=stats, - folder_metadata=metadata, - assets_path=assets_path, - source_dir=str(web_dir), - metadata_file_path=get_metadata_file_path(web_dir) - ) - f.write(rendered_html) - - print(f"Generated {output_html}") - - -def convert_pdf_to_png(pdf_path: Path) -> None: - """ - Convert a PDF file to PNG format using ImageMagick. - - Only converts if the PNG doesn't exist or if the PDF is newer than - the PNG (with a 30-second buffer to handle filesystem timing issues). - - Args: - pdf_path: Path to the source PDF file - - Raises: - subprocess.CalledProcessError: If ImageMagick conversion fails - """ - png_path = pdf_path.with_suffix(".png") - - if png_path.exists(): - pdf_mtime = pdf_path.stat().st_mtime - png_mtime = png_path.stat().st_mtime - if png_mtime >= (pdf_mtime + 30): - return - else: - print(f"PDF {pdf_path.name} is newer than PNG, reconverting...") - - print(f"Converting {pdf_path} to PNG") - subprocess.run([ - "convert", - "-density", str(CONFIG.png_dpi), - str(pdf_path), - "-quality", "95", - str(png_path) - ], check=True) - - -def needs_update(source_file: Path, target_file: Path) -> bool: - """ - Check if target file needs updating based on source modification time. - - Args: - source_file: Path to the source file - target_file: Path to the target file - - Returns: - True if target needs update, False otherwise - """ - if not target_file.exists(): - return True - - source_mtime = source_file.stat().st_mtime - target_mtime = target_file.stat().st_mtime - - return source_mtime > (target_mtime + 30) - - def build_gallery( source_dir: Path, web_dir: Path, @@ -294,7 +89,12 @@ def build_gallery( # Process all plot files (PDFs and HTMLs) for plot_file in plot_files: - item = process_plot_files(plot_file, web_dir, current_metadata) + item = process_plot_files( + CONFIG=CONFIG, + plot_file=plot_file, + web_dir=web_dir, + current_metadata=current_metadata, + ) items.append(item) plot_metadata_cache[plot_file.stem] = item["metadata"] @@ -310,6 +110,8 @@ def build_gallery( subdir_names.append(subdir.name) render_gallery_page( + CONFIG=CONFIG, + template=template, web_dir=web_dir, items=items, subdirs=subdir_names, @@ -318,66 +120,6 @@ def build_gallery( ) -def calculate_directory_stats(directory: Path) -> dict: - """ - Calculate statistics for a directory. - - Args: - directory: Path to the directory to analyze - - Returns: - Dictionary containing file count, folder count, and total size - """ - stats = { - "file_count": 0, - "folder_count": 0, - "total_size": 0, - "pdf_size": 0, - "png_size": 0, - } - - if not directory.exists(): - return stats - - for item in directory.rglob("*"): - if item.is_file(): - stats["file_count"] += 1 - size = item.stat().st_size - stats["total_size"] += size - - if item.suffix.lower() == '.pdf': - stats["pdf_size"] += size - elif item.suffix.lower() == '.png': - stats["png_size"] += size - elif item.is_dir(): - stats["folder_count"] += 1 - - return stats - - -def format_file_size(size_bytes: int) -> str: - """ - Format file size in human readable format. - - Args: - size_bytes: Size in bytes - - Returns: - Formatted size string - """ - if size_bytes == 0: - return "0 B" - - size_names = ["B", "KB", "MB", "GB", "TB"] - size = float(size_bytes) - i = 0 - while size >= 1024 and i < len(size_names) - 1: - size /= 1024 - i += 1 - - return f"{size:.1f} {size_names[i]}" - - def main(clean_first: bool = False) -> None: """ Main entry point for gallery generation. @@ -418,8 +160,14 @@ def main(clean_first: bool = False) -> None: source_subdirs.append(source.name) if source_path.is_file() and source_path.suffix == '.pdf': - item = process_plot_files(source_path, source_web_dir) + item = process_plot_files( + CONFIG=CONFIG, + plot_file=source_path, + web_dir=source_web_dir, + ) render_gallery_page( + CONFIG=CONFIG, + template=template, web_dir=source_web_dir, items=[item], subdirs=[], @@ -436,6 +184,8 @@ def main(clean_first: bool = False) -> None: print(f"Processed {source.name}: {source.path}") render_gallery_page( + CONFIG=CONFIG, + template=template, web_dir=gallery_root, items=[], subdirs=source_subdirs, diff --git a/utils/processing.py b/utils/processing.py new file mode 100644 index 0000000..8407ecd --- /dev/null +++ b/utils/processing.py @@ -0,0 +1,230 @@ +import shutil +from typing import Any, Dict +from pathlib import Path +import subprocess +from jinja2 import Template + +from utils.metadata import ( + resolve_metadata_for_plot, + get_metadata_file_path, +) +from utils.stats import ( + calculate_directory_stats, + format_file_size, +) +from utils.config import Config + + +def process_html_file( + html_file: Path, + web_dir: Path, + current_metadata: Dict[str, Any] = None +) -> dict: + """ + Process HTML plot file, copying it to web directory. + + Args: + html_file: Path to the source HTML file + web_dir: Target web directory + current_metadata: Current metadata dictionary for the plot + + Returns: + Dictionary containing plot information + """ + web_html = web_dir / html_file.name + + if needs_update(html_file, web_html): + shutil.copy2(html_file, web_html) + else: + print(f"Skipping {html_file.name} (up to date)") + + # Get source file creation time + source_creation_time = int(html_file.stat().st_ctime) + + # Resolve metadata if provided + plot_metadata = {} + if current_metadata is not None: + plot_metadata = resolve_metadata_for_plot(html_file, current_metadata) + + return { + "name": html_file.stem, + "html_href": html_file.name, + "is_html": True, + "metadata": plot_metadata, + "creation_time": source_creation_time + } + + +def process_plot_files( + CONFIG: Config, + plot_file: Path, + web_dir: Path, + current_metadata: Dict[str, Any] = None, +) -> dict: + """ + Process plot files (PDF/PNG or HTML), handling conversion and copying. + + Args: + plot_file: Path to the source plot file (PDF or HTML) + web_dir: Target web directory + current_metadata: Current metadata dictionary for the plot + + Returns: + Dictionary containing plot information + """ + if plot_file.suffix.lower() == '.html': + return process_html_file(plot_file, web_dir, current_metadata) + + # Handle PDF files + png_file = plot_file.with_suffix(".png") + web_pdf = web_dir / plot_file.name + web_png = web_dir / png_file.name + + if needs_update(plot_file, web_pdf): + shutil.copy2(plot_file, web_pdf) + else: + print(f"Skipping {plot_file.name} (up to date)") + + if not png_file.exists(): + convert_pdf_to_png(plot_file, CONFIG=CONFIG) + + if needs_update(png_file, web_png): + shutil.copy2(png_file, web_png) + else: + print(f"Skipping {png_file.name} (up to date)") + + source_creation_time = int(plot_file.stat().st_ctime) + + plot_metadata = {} + if current_metadata is not None: + plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata) + + return { + "name": plot_file.stem, + "pdf_href": plot_file.name, + "png_href": png_file.name, + "is_html": False, + "metadata": plot_metadata, + "creation_time": source_creation_time + } + + +def render_gallery_page( + CONFIG: Config, + template: Template, + web_dir: Path, + items: list, + subdirs: list, + relative_path: Path, + title: str = None, + metadata: dict = None +) -> None: + """ + Unified template rendering for all gallery pages. + + Args: + web_dir: Target web directory + items: List of plot items + subdirs: List of subdirectory names + relative_path: Relative path from gallery root + title: Page title (optional) + metadata: Metadata dictionary (optional) + """ + if title is None: + title = "Gallery" if relative_path == Path( + ".") else f"Gallery: {relative_path}" + + if metadata is None: + metadata = {} + + # Calculate statistics + current_stats = calculate_directory_stats(web_dir) + stats = { + "file_count": len(items), + "folder_count": len(subdirs), + "total_size": format_file_size(current_stats["total_size"]), + "total_size_bytes": current_stats["total_size"] + } + + # Calculate relative path to assets + if relative_path == Path("."): + assets_path = "../assets" + else: + depth = len(relative_path.parts) + assets_path = "../" * (depth + 1) + "assets" + + # For root level, show only directory structure + if relative_path == Path("."): + items = [] + + output_html = web_dir / "index.html" + with output_html.open("w") as f: + rendered_html = template.render( + title=title, + items=items, + subdirs=subdirs, + relpath=str(relative_path), + paths=CONFIG.paths, + ui=CONFIG.ui, + stats=stats, + folder_metadata=metadata, + assets_path=assets_path, + source_dir=str(web_dir), + metadata_file_path=get_metadata_file_path(web_dir) + ) + f.write(rendered_html) + + print(f"Generated {output_html}") + + +def convert_pdf_to_png(pdf_path: Path, CONFIG: Config) -> None: + """ + Convert a PDF file to PNG format using ImageMagick. + + Only converts if the PNG doesn't exist or if the PDF is newer than + the PNG (with a 30-second buffer to handle filesystem timing issues). + + Args: + pdf_path: Path to the source PDF file + + Raises: + subprocess.CalledProcessError: If ImageMagick conversion fails + """ + png_path = pdf_path.with_suffix(".png") + + if png_path.exists(): + pdf_mtime = pdf_path.stat().st_mtime + png_mtime = png_path.stat().st_mtime + if png_mtime >= (pdf_mtime + 30): + return + else: + print(f"PDF {pdf_path.name} is newer than PNG, reconverting...") + + print(f"Converting {pdf_path} to PNG") + subprocess.run([ + "convert", + "-density", str(CONFIG.png_dpi), + str(pdf_path), + "-quality", "95", + str(png_path) + ], check=True) + + +def needs_update(source_file: Path, target_file: Path) -> bool: + """ + Check if target file needs updating based on source modification time. + + Args: + source_file: Path to the source file + target_file: Path to the target file + + Returns: + True if target needs update, False otherwise + """ + if not target_file.exists(): + return True + + source_mtime = source_file.stat().st_mtime + target_mtime = target_file.stat().st_mtime + + return source_mtime > (target_mtime + 30) diff --git a/utils/stats.py b/utils/stats.py new file mode 100644 index 0000000..d7d8ea4 --- /dev/null +++ b/utils/stats.py @@ -0,0 +1,62 @@ +from pathlib import Path + + +def calculate_directory_stats(directory: Path) -> dict: + """ + Calculate statistics for a directory. + + Args: + directory: Path to the directory to analyze + + Returns: + Dictionary containing file count, folder count, and total size + """ + stats = { + "file_count": 0, + "folder_count": 0, + "total_size": 0, + "pdf_size": 0, + "png_size": 0, + } + + if not directory.exists(): + return stats + + for item in directory.rglob("*"): + if item.is_file(): + stats["file_count"] += 1 + size = item.stat().st_size + stats["total_size"] += size + + if item.suffix.lower() == '.pdf': + stats["pdf_size"] += size + elif item.suffix.lower() == '.png': + stats["png_size"] += size + elif item.is_dir(): + stats["folder_count"] += 1 + + return stats + + +def format_file_size(size_bytes: int) -> str: + """ + Format file size in human readable format. + + Args: + size_bytes: Size in bytes + + Returns: + Formatted size string + """ + if size_bytes == 0: + return "0 B" + + size_names = ["B", "KB", "MB", "GB", "TB"] + size = float(size_bytes) + i = 0 + while size >= 1024 and i < len(size_names) - 1: + size /= 1024 + i += 1 + + return f"{size:.1f} {size_names[i]}" +