diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 994d98c..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,31 +0,0 @@ -# You can override the included template(s) by including variable overrides -# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Secret Detection customization: https://docs.gitlab.com/user/application_security/secret_detection/pipeline/configure -# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings -# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence -stages: -- build -- test -- deploy -- review -- dast -- staging -- canary -- production -- incremental rollout 10% -- incremental rollout 25% -- incremental rollout 50% -- incremental rollout 100% -- performance -- cleanup -- secret-detection -sast: - stage: test -include: -- template: Auto-DevOps.gitlab-ci.yml -variables: - SECRET_DETECTION_ENABLED: 'true' -secret_detection: - stage: secret-detection diff --git a/config.yaml b/config.yaml index 14ef13b..e6c4910 100644 --- a/config.yaml +++ b/config.yaml @@ -47,5 +47,5 @@ metadata: sources: - name: "ttbar_analysis" path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/" - - name: "NEEDLE_benchmarks" + - name: "needle_benchmarks" path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots" diff --git a/debug_template.py b/debug_template.py new file mode 100644 index 0000000..89e9c8a --- /dev/null +++ b/debug_template.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +from pathlib import Path +from jinja2 import Environment, FileSystemLoader + +# Test template rendering +env = Environment(loader=FileSystemLoader(".")) +template = env.get_template("templates/gallery.html") + +# Mock data +test_items = [{ + "name": "test_plot", + "pdf_href": "test_plot.pdf", + "png_href": "test_plot.png", + "metadata": {"key1": "value1", "key2": "value2"} +}] + +test_config = { + "paths": {"work_dir": "/work/kschmidt/web"}, + "ui": {"search_debounce_ms": 300, "max_recent_plots": 20} +} + +rendered = template.render( + title="Test Gallery", + items=test_items, + subdirs=["test_subdir"], + relpath="test/path", + paths=test_config["paths"], + ui=test_config["ui"], + stats={"file_count": 1, "folder_count": 1, "total_size": "100 KB", "total_size_bytes": 100000}, + folder_metadata={}, + assets_path="../assets" +) + +# Extract just the metadata button part +lines = rendered.split('\n') +for i, line in enumerate(lines): + if 'metadata-btn' in line: + print(f"Line {i}: {line.strip()}") + if i > 0: + print(f"Line {i-1}: {lines[i-1].strip()}") + if i < len(lines)-1: + print(f"Line {i+1}: {lines[i+1].strip()}") + break + +print("\n--- Assets path in template ---") +for i, line in enumerate(lines): + if 'assets_path' in line: + print(f"Line {i}: {line.strip()}") diff --git a/generate_gallery.py b/generate_gallery.py index ea93ba1..e47e60f 100644 --- a/generate_gallery.py +++ b/generate_gallery.py @@ -15,6 +15,8 @@ Features: import subprocess import shutil +import os +import sys from pathlib import Path from typing import Dict, Any, Optional from jinja2 import Environment, FileSystemLoader @@ -95,8 +97,7 @@ def build_gallery(source_dir: Path, web_dir: Path, Processes all PDF files in the source directory, converts them to PNG, copies both to the web directory, and generates index.html files with - navigation and thumbnails. HTML files are always regenerated to ensure - that new subdirectories appear in navigation immediately. + navigation and thumbnails. Now includes metadata support. Args: source_dir: Source directory containing PDF files @@ -115,15 +116,12 @@ def build_gallery(source_dir: Path, web_dir: Path, current_metadata = merge_metadata(inherited_metadata, folder_metadata) pdf_files = list(source_dir.glob("*.pdf")) - png_files = list(source_dir.glob("*.png")) subdirs = [d for d in source_dir.iterdir() if d.is_dir()] items = [] plot_metadata_cache = {} - processed_stems = set() - # First, process PDF files (with PNG conversion) for pdf_file in pdf_files: png_file = pdf_file.with_suffix(".png") @@ -147,12 +145,6 @@ def build_gallery(source_dir: Path, web_dir: Path, # Resolve metadata for this specific plot plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata) - if not plot_metadata: - plot_metadata = {} - for k, v in plot_metadata.items(): - if isinstance(v, str): - plot_metadata[k] = (v.replace('<', '').replace('>', '') - .replace('&', '').replace('"', "'")) plot_metadata_cache[pdf_file.stem] = plot_metadata items.append({ @@ -161,40 +153,6 @@ def build_gallery(source_dir: Path, web_dir: Path, "png_href": png_file.name, "metadata": plot_metadata }) - processed_stems.add(pdf_file.stem) - - # Second, process standalone PNG files (no corresponding PDF) - for png_file in png_files: - # Skip if we already processed this PNG via its PDF - if png_file.stem in processed_stems: - continue - - web_png = web_dir / png_file.name - - if needs_update(png_file, web_png): - print(f"Copying {png_file} to {web_png}") - shutil.copy2(png_file, web_png) - else: - print(f"Skipping {png_file.name} (up to date)") - - # Resolve metadata for this standalone PNG plot - plot_metadata = resolve_metadata_for_plot(png_file, current_metadata) - if not plot_metadata: - plot_metadata = {} - for k, v in plot_metadata.items(): - if isinstance(v, str): - plot_metadata[k] = (v.replace('<', '').replace('>', '') - .replace('&', '').replace('"', "'")) - plot_metadata_cache[png_file.stem] = plot_metadata - - # For standalone PNGs, link to the PNG for both thumbnail and main view - items.append({ - "name": png_file.stem, - "pdf_href": png_file.name, # Link to PNG instead of PDF - "png_href": png_file.name, # Thumbnail is also the PNG - "metadata": plot_metadata - }) - processed_stems.add(png_file.stem) # Save metadata cache for this directory save_metadata_cache(web_dir, plot_metadata_cache) @@ -209,6 +167,17 @@ def build_gallery(source_dir: Path, web_dir: Path, subdir_names.append(subdir.name) output_html = web_dir / "index.html" + + # Always regenerate HTML to ensure subdirectory changes are reflected + # This ensures new subdirectories appear in navigation + force_regeneration = True + if output_html.exists() and not force_regeneration: + html_mtime = output_html.stat().st_mtime + # Check if any subdirectory is newer than the HTML file + for subdir in subdirs: + if subdir.stat().st_mtime > html_mtime: + print(f"Subdirectory {subdir.name} is newer, forcing regeneration") + break if relative_path == Path("."): title = "Gallery" @@ -309,62 +278,19 @@ def format_file_size(size_bytes: int) -> str: return f"{size:.1f} {size_names[i]}" -def generate_root_index(gallery_root: Path) -> None: +def refresh_gallery_cgi(): """ - Generate the main index.html file for the gallery root. - - This serves as the entry point to navigate to all configured sources. - - Args: - gallery_root: Path to the gallery root directory + CGI handler to refresh the gallery from a web request. + Outputs a minimal HTTP response and triggers gallery regeneration. """ - # Collect information about all source directories - subdirs = [] - total_plots = 0 - total_size = 0 - - for source in CONFIG.sources: - source_web_dir = gallery_root / source.name - if source_web_dir.exists(): - subdirs.append(source.name) - - # Count plots and calculate size for this source - stats = calculate_directory_stats(source_web_dir) - total_plots += stats.get("file_count", 0) - total_size += stats.get("total_size", 0) - - # Calculate overall statistics - stats = { - "file_count": total_plots, - "folder_count": len(subdirs), - "total_size": format_file_size(total_size), - "total_size_bytes": total_size - } - - # Assets are at the same level as gallery for root - assets_path = "../assets" - - output_html = gallery_root / "index.html" - with output_html.open("w") as f: - rendered_html = template.render( - title="Scientific Gallery", - items=[], # No individual plots at root level - subdirs=subdirs, - relpath=".", - paths=CONFIG.paths, - ui=CONFIG.ui, - stats=stats, - folder_metadata={ - "description": ("Main gallery containing scientific plots " - "and analyses"), - "sources": [{"name": s.name, "path": s.path} - for s in CONFIG.sources] - }, - assets_path=assets_path - ) - f.write(rendered_html) - - print(f"Generated root gallery index: {output_html}") + import traceback + print("Content-Type: text/plain\n") + try: + main() + print("Gallery refreshed successfully.") + except Exception as e: + print(f"Error refreshing gallery: {e}") + traceback.print_exc(file=sys.stdout) def main(clean_first: bool = False) -> None: @@ -478,65 +404,6 @@ def main(clean_first: bool = False) -> None: print(f"Generated {output_html}") print(f"Processed {source.name}: {source.path}") - elif source_path.is_file() and source_path.suffix == '.png': - # Handle standalone PNG files - source_web_dir = gallery_root / source.name - source_web_dir.mkdir(parents=True, exist_ok=True) - - png_name = source_path.name - web_png_path = source_web_dir / png_name - - if needs_update(source_path, web_png_path): - print(f"Copying {source_path} to {web_png_path}") - shutil.copy2(source_path, web_png_path) - else: - print(f"Skipping {source_path.name} (up to date)") - - # Resolve metadata for this single PNG plot - plot_metadata = resolve_metadata_for_plot(source_path, {}) - plot_metadata_cache[source_path.stem] = plot_metadata - - items = [{ - "name": source_path.stem, - "pdf_href": png_name, # Link to PNG instead of PDF - "png_href": png_name, # Thumbnail is the PNG - "metadata": plot_metadata - }] - - # Save metadata cache for this directory - plot_cache = {source_path.stem: plot_metadata} - save_metadata_cache(source_web_dir, plot_cache) - - # Calculate statistics for single file - current_stats = calculate_directory_stats(source_web_dir) - stats = { - "file_count": 1, - "folder_count": 0, - "total_size": format_file_size(current_stats["total_size"]), - "total_size_bytes": current_stats["total_size"] - } - - # Calculate relative path to assets for single file - # Single files are at depth 1 (gallery_root/source.name/index.html) - assets_path = "../assets" - - output_html = source_web_dir / "index.html" - with output_html.open("w") as f: - f.write(template.render( - title=source.name, - items=items, - subdirs=[], - relpath=source.name, - paths=CONFIG.paths, - ui=CONFIG.ui, - stats=stats, - folder_metadata={}, - assets_path=assets_path - )) - - print(f"Generated {output_html}") - print(f"Processed {source.name}: {source.path}") - elif source_path.is_dir(): source_web_dir = gallery_root / source.name source_web_dir.mkdir(parents=True, exist_ok=True) @@ -544,20 +411,22 @@ def main(clean_first: bool = False) -> None: print(f"Processed {source.name}: {source.path}") else: print(f"Warning: Source {source.path} is neither a " - f"directory nor a PDF/PNG file") + f"directory nor a PDF file") print("Done") - - # Generate root index.html for gallery navigation - generate_root_index(gallery_root) - - # No copying of this script to CGI location if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser(description='Generate gallery') - parser.add_argument('--clean', action='store_true', - help='Clean gallery directory before generation') - args = parser.parse_args() - main(clean_first=args.clean) + if 'GATEWAY_INTERFACE' in os.environ: + refresh_gallery_cgi() + else: + import argparse + + parser = argparse.ArgumentParser(description='Generate gallery') + parser.add_argument( + '--clean', + action='store_true', + help='Clean gallery directory before generation' + ) + args = parser.parse_args() + main(clean_first=args.clean) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b519773 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "plot-gallery" +version = "0.1.0" +description = "Host your plots on a personal website" +authors = [ + { name = "K. Schmidt" } +] +readme = "README.md" +requires-python = ">=3.9" + +[tool.black] +line-length = 120 +target-version = ['py39'] + +[tool.isort] +profile = "black" +line_length = 120 + +[tool.flake8] +max-line-length = 120 +extend-ignore = ["E203", "W503"] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" \ No newline at end of file