Kylian Schmidt
dbd67f6039
- Fix template rendering by storing rendered HTML in variable before writing This resolves subdirectories not appearing in navigation despite being correctly detected and included in template variables - Add missing template variables (paths.work_dir) to gallery config - Add CGI refresh handler for web-based gallery regeneration - Improve PDF export functionality with better error handling - Add ESC key shortcut documentation for exiting selection mode - Enhanced export manager with proper dependency checking Fixes issue where new subdirectories were detected but not displayed in browser navigation due to incomplete template rendering.
408 lines
12 KiB
Python
408 lines
12 KiB
Python
"""
|
|
Scientific Gallery Generator
|
|
|
|
This module generates static HTML galleries from scientific plot collections.
|
|
It converts PDF plots to PNG thumbnails, creates responsive web interfaces,
|
|
and organizes plots into hierarchical directory structures.
|
|
|
|
Features:
|
|
- PDF to PNG conversion with configurable DPI
|
|
- Incremental updates (only converts when source is newer)
|
|
- Jinja2 templating for consistent HTML generation
|
|
- Support for nested folder structures
|
|
- Responsive grid layout with search and navigation
|
|
"""
|
|
|
|
import subprocess
|
|
import shutil
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional
|
|
from jinja2 import Environment, FileSystemLoader
|
|
|
|
from orchestration.config import Config
|
|
from orchestration.metadata import (
|
|
load_folder_metadata,
|
|
merge_metadata,
|
|
resolve_metadata_for_plot,
|
|
save_metadata_cache
|
|
)
|
|
|
|
|
|
CONFIG = Config.from_yaml("config.yaml")
|
|
|
|
env = Environment(loader=FileSystemLoader("."))
|
|
template = env.get_template("templates/gallery.html")
|
|
|
|
|
|
def convert_pdf_to_png(pdf_path: Path) -> None:
|
|
"""
|
|
Convert a PDF file to PNG format using ImageMagick.
|
|
|
|
Only converts if the PNG doesn't exist or if the PDF is newer than
|
|
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
|
|
|
Args:
|
|
pdf_path: Path to the source PDF file
|
|
|
|
Raises:
|
|
subprocess.CalledProcessError: If ImageMagick conversion fails
|
|
"""
|
|
png_path = pdf_path.with_suffix(".png")
|
|
|
|
if png_path.exists():
|
|
pdf_mtime = pdf_path.stat().st_mtime
|
|
png_mtime = png_path.stat().st_mtime
|
|
if png_mtime >= (pdf_mtime + 30):
|
|
return
|
|
else:
|
|
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
|
|
|
|
print(f"Converting {pdf_path} → {png_path}")
|
|
subprocess.run([
|
|
"convert",
|
|
"-density", str(CONFIG.png_dpi),
|
|
str(pdf_path),
|
|
"-quality", "95",
|
|
str(png_path)
|
|
], check=True)
|
|
|
|
|
|
def needs_update(source_file: Path, target_file: Path) -> bool:
|
|
"""
|
|
Check if target file needs updating based on source modification time.
|
|
|
|
Args:
|
|
source_file: Path to the source file
|
|
target_file: Path to the target file
|
|
|
|
Returns:
|
|
True if target needs update, False otherwise
|
|
"""
|
|
if not target_file.exists():
|
|
return True
|
|
|
|
source_mtime = source_file.stat().st_mtime
|
|
target_mtime = target_file.stat().st_mtime
|
|
|
|
return source_mtime > (target_mtime + 30)
|
|
|
|
|
|
def build_gallery(source_dir: Path, web_dir: Path,
|
|
relative_path: Path = None,
|
|
inherited_metadata: Optional[Dict[str, Any]] = None) -> None:
|
|
"""
|
|
Recursively build gallery structure from source directory.
|
|
|
|
Processes all PDF files in the source directory, converts them to PNG,
|
|
copies both to the web directory, and generates index.html files with
|
|
navigation and thumbnails. Now includes metadata support.
|
|
|
|
Args:
|
|
source_dir: Source directory containing PDF files
|
|
web_dir: Target web directory for gallery output
|
|
relative_path: Relative path from gallery root (for navigation)
|
|
inherited_metadata: Metadata inherited from parent directories
|
|
"""
|
|
if relative_path is None:
|
|
relative_path = Path(".")
|
|
|
|
if inherited_metadata is None:
|
|
inherited_metadata = {}
|
|
|
|
# Load folder-level metadata and merge with inherited metadata
|
|
folder_metadata = load_folder_metadata(source_dir)
|
|
current_metadata = merge_metadata(inherited_metadata, folder_metadata)
|
|
|
|
pdf_files = list(source_dir.glob("*.pdf"))
|
|
|
|
subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
|
|
|
|
items = []
|
|
plot_metadata_cache = {}
|
|
|
|
for pdf_file in pdf_files:
|
|
png_file = pdf_file.with_suffix(".png")
|
|
|
|
web_pdf = web_dir / pdf_file.name
|
|
web_png = web_dir / png_file.name
|
|
|
|
if needs_update(pdf_file, web_pdf):
|
|
print(f"Copying {pdf_file} to {web_pdf}")
|
|
shutil.copy2(pdf_file, web_pdf)
|
|
else:
|
|
print(f"Skipping {pdf_file.name} (up to date)")
|
|
|
|
if not png_file.exists():
|
|
convert_pdf_to_png(pdf_file)
|
|
|
|
if needs_update(png_file, web_png):
|
|
print(f"Copying {png_file} to {web_png}")
|
|
shutil.copy2(png_file, web_png)
|
|
else:
|
|
print(f"Skipping {png_file.name} (up to date)")
|
|
|
|
# Resolve metadata for this specific plot
|
|
plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
|
|
plot_metadata_cache[pdf_file.stem] = plot_metadata
|
|
|
|
items.append({
|
|
"name": pdf_file.stem,
|
|
"pdf_href": pdf_file.name,
|
|
"png_href": png_file.name,
|
|
"metadata": plot_metadata
|
|
})
|
|
|
|
# Save metadata cache for this directory
|
|
save_metadata_cache(web_dir, plot_metadata_cache)
|
|
|
|
subdir_names = []
|
|
for subdir in subdirs:
|
|
subdir_web = web_dir / subdir.name
|
|
subdir_web.mkdir(exist_ok=True)
|
|
subdir_relative = relative_path / subdir.name
|
|
# Pass current metadata to subdirectories
|
|
build_gallery(subdir, subdir_web, subdir_relative, current_metadata)
|
|
subdir_names.append(subdir.name)
|
|
|
|
output_html = web_dir / "index.html"
|
|
|
|
if relative_path == Path("."):
|
|
title = "Gallery"
|
|
else:
|
|
title = f"Gallery: {relative_path}"
|
|
|
|
# Calculate statistics for current directory
|
|
current_stats = calculate_directory_stats(web_dir)
|
|
stats = {
|
|
"file_count": len(items),
|
|
"folder_count": len(subdir_names),
|
|
"total_size": format_file_size(current_stats["total_size"]),
|
|
"total_size_bytes": current_stats["total_size"]
|
|
}
|
|
|
|
# Calculate relative path to assets based on directory depth
|
|
if relative_path == Path("."):
|
|
assets_path = "../assets"
|
|
else:
|
|
# Count directory levels to go back to gallery, then to public_html
|
|
depth = len(relative_path.parts)
|
|
assets_path = "../" * (depth + 1) + "assets"
|
|
|
|
with output_html.open("w") as f:
|
|
rendered_html = template.render(
|
|
title=title,
|
|
items=items,
|
|
subdirs=subdir_names,
|
|
relpath=str(relative_path),
|
|
paths=CONFIG.paths,
|
|
ui=CONFIG.ui,
|
|
stats=stats,
|
|
folder_metadata=current_metadata,
|
|
assets_path=assets_path
|
|
)
|
|
f.write(rendered_html)
|
|
|
|
print(f"Generated {output_html}")
|
|
|
|
|
|
def calculate_directory_stats(directory: Path) -> dict:
|
|
"""
|
|
Calculate statistics for a directory.
|
|
|
|
Args:
|
|
directory: Path to the directory to analyze
|
|
|
|
Returns:
|
|
Dictionary containing file count, folder count, and total size
|
|
"""
|
|
stats = {
|
|
"file_count": 0,
|
|
"folder_count": 0,
|
|
"total_size": 0,
|
|
"pdf_size": 0,
|
|
"png_size": 0
|
|
}
|
|
|
|
if not directory.exists():
|
|
return stats
|
|
|
|
for item in directory.rglob("*"):
|
|
if item.is_file():
|
|
stats["file_count"] += 1
|
|
size = item.stat().st_size
|
|
stats["total_size"] += size
|
|
|
|
if item.suffix.lower() == '.pdf':
|
|
stats["pdf_size"] += size
|
|
elif item.suffix.lower() == '.png':
|
|
stats["png_size"] += size
|
|
elif item.is_dir():
|
|
stats["folder_count"] += 1
|
|
|
|
return stats
|
|
|
|
|
|
def format_file_size(size_bytes: int) -> str:
|
|
"""
|
|
Format file size in human readable format.
|
|
|
|
Args:
|
|
size_bytes: Size in bytes
|
|
|
|
Returns:
|
|
Formatted size string
|
|
"""
|
|
if size_bytes == 0:
|
|
return "0 B"
|
|
|
|
size_names = ["B", "KB", "MB", "GB", "TB"]
|
|
size = float(size_bytes)
|
|
i = 0
|
|
while size >= 1024 and i < len(size_names) - 1:
|
|
size /= 1024
|
|
i += 1
|
|
|
|
return f"{size:.1f} {size_names[i]}"
|
|
|
|
|
|
def refresh_gallery_cgi():
|
|
"""
|
|
CGI handler to refresh the gallery from a web request.
|
|
Outputs a minimal HTTP response and triggers gallery regeneration.
|
|
"""
|
|
import traceback
|
|
print("Content-Type: text/plain\n")
|
|
try:
|
|
main()
|
|
print("Gallery refreshed successfully.")
|
|
except Exception as e:
|
|
print(f"Error refreshing gallery: {e}")
|
|
traceback.print_exc(file=sys.stdout)
|
|
|
|
|
|
def main() -> None:
|
|
"""
|
|
Main entry point for gallery generation.
|
|
|
|
Processes all configured sources and generates the complete gallery
|
|
structure in the web directory. Cleans up the gallery directory
|
|
on first run to ensure consistency.
|
|
"""
|
|
gallery_root = Path(CONFIG.web_folder) / CONFIG.plot_root
|
|
|
|
if gallery_root.exists():
|
|
print(f"Gallery directory {gallery_root} exists, cleaning up...")
|
|
shutil.rmtree(gallery_root)
|
|
|
|
gallery_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Copy assets folder to the gallery root
|
|
assets_src = Path("assets")
|
|
assets_dst = gallery_root.parent / "assets"
|
|
|
|
if assets_src.exists():
|
|
if assets_dst.exists():
|
|
shutil.rmtree(assets_dst)
|
|
shutil.copytree(assets_src, assets_dst)
|
|
print(f"Copied assets from {assets_src} to {assets_dst}")
|
|
else:
|
|
print(f"Warning: Assets directory {assets_src} not found")
|
|
|
|
plot_metadata_cache = {}
|
|
|
|
for source in CONFIG.sources:
|
|
source_path = Path(source.path)
|
|
|
|
if source_path.is_file() and source_path.suffix == '.pdf':
|
|
source_web_dir = gallery_root / source.name
|
|
source_web_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
pdf_name = source_path.name
|
|
png_name = source_path.with_suffix('.png').name
|
|
|
|
web_pdf_path = source_web_dir / pdf_name
|
|
web_png_path = source_web_dir / png_name
|
|
source_png_path = source_path.with_suffix('.png')
|
|
|
|
if needs_update(source_path, web_pdf_path):
|
|
print(f"Copying {source_path} to {web_pdf_path}")
|
|
shutil.copy2(source_path, web_pdf_path)
|
|
else:
|
|
print(f"Skipping {source_path.name} (up to date)")
|
|
|
|
if not source_png_path.exists():
|
|
convert_pdf_to_png(source_path)
|
|
|
|
if needs_update(source_png_path, web_png_path):
|
|
print(f"Copying {source_png_path} to {web_png_path}")
|
|
shutil.copy2(source_png_path, web_png_path)
|
|
else:
|
|
print(f"Skipping {source_png_path.name} (up to date)")
|
|
|
|
# Resolve metadata for this single plot
|
|
plot_metadata = resolve_metadata_for_plot(source_path, {})
|
|
plot_metadata_cache[source_path.stem] = plot_metadata
|
|
|
|
items = [{
|
|
"name": source_path.stem,
|
|
"pdf_href": pdf_name,
|
|
"png_href": png_name,
|
|
"metadata": plot_metadata
|
|
}]
|
|
|
|
# Save metadata cache for this directory
|
|
plot_cache = {source_path.stem: plot_metadata}
|
|
save_metadata_cache(source_web_dir, plot_cache)
|
|
|
|
# Calculate statistics for single file
|
|
current_stats = calculate_directory_stats(source_web_dir)
|
|
stats = {
|
|
"file_count": 1,
|
|
"folder_count": 0,
|
|
"total_size": format_file_size(current_stats["total_size"]),
|
|
"total_size_bytes": current_stats["total_size"]
|
|
}
|
|
|
|
# Calculate relative path to assets for single file
|
|
# Single files are at depth 1 (gallery_root/source.name/index.html)
|
|
assets_path = "../assets"
|
|
|
|
output_html = source_web_dir / "index.html"
|
|
with output_html.open("w") as f:
|
|
f.write(template.render(
|
|
title=source.name,
|
|
items=items,
|
|
subdirs=[],
|
|
relpath=source.name,
|
|
paths=CONFIG.paths,
|
|
ui=CONFIG.ui,
|
|
stats=stats,
|
|
folder_metadata={},
|
|
assets_path=assets_path
|
|
))
|
|
|
|
print(f"Generated {output_html}")
|
|
print(f"Processed {source.name}: {source.path}")
|
|
|
|
elif source_path.is_dir():
|
|
source_web_dir = gallery_root / source.name
|
|
source_web_dir.mkdir(parents=True, exist_ok=True)
|
|
build_gallery(source_path, source_web_dir, Path(source.name))
|
|
print(f"Processed {source.name}: {source.path}")
|
|
else:
|
|
print(f"Warning: Source {source.path} is neither a "
|
|
f"directory nor a PDF file")
|
|
|
|
print("Done")
|
|
# No copying of this script to CGI location
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# If run as CGI, call the CGI handler
|
|
if 'GATEWAY_INTERFACE' in os.environ:
|
|
refresh_gallery_cgi()
|
|
else:
|
|
main()
|