Version 0.1.0
This commit is contained in:
Kylian Schmidt
@@ -0,0 +1,448 @@
|
||||
"""
|
||||
Scientific Gallery Generator
|
||||
|
||||
This module generates static HTML galleries from scientific plot collections.
|
||||
It converts PDF plots to PNG thumbnails, creates responsive web interfaces,
|
||||
and organizes plots into hierarchical directory structures.
|
||||
|
||||
Features:
|
||||
- PDF to PNG conversion with configurable DPI
|
||||
- Incremental updates (only converts when source is newer)
|
||||
- Jinja2 templating for consistent HTML generation
|
||||
- Support for nested folder structures
|
||||
- Responsive grid layout with search and navigation
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import shutil
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
from orchestration.config import Config
|
||||
from orchestration.metadata import (
|
||||
load_folder_metadata,
|
||||
merge_metadata,
|
||||
resolve_metadata_for_plot,
|
||||
save_metadata_cache,
|
||||
get_metadata_file_path
|
||||
)
|
||||
|
||||
|
||||
CONFIG = Config.from_yaml("config.yaml")
|
||||
|
||||
|
||||
def datetime_from_timestamp(timestamp):
|
||||
"""Convert a Unix timestamp to a datetime object."""
|
||||
return datetime.fromtimestamp(timestamp)
|
||||
|
||||
|
||||
def strftime_filter(dt, fmt):
|
||||
"""Format a datetime object using strftime."""
|
||||
return dt.strftime(fmt)
|
||||
|
||||
|
||||
env = Environment(loader=FileSystemLoader("."))
|
||||
env.filters['datetime_from_timestamp'] = datetime_from_timestamp
|
||||
env.filters['strftime'] = strftime_filter
|
||||
template = env.get_template("templates/gallery.html")
|
||||
|
||||
|
||||
def convert_pdf_to_png(pdf_path: Path) -> None:
|
||||
"""
|
||||
Convert a PDF file to PNG format using ImageMagick.
|
||||
|
||||
Only converts if the PNG doesn't exist or if the PDF is newer than
|
||||
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
||||
|
||||
Args:
|
||||
pdf_path: Path to the source PDF file
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If ImageMagick conversion fails
|
||||
"""
|
||||
png_path = pdf_path.with_suffix(".png")
|
||||
|
||||
if png_path.exists():
|
||||
pdf_mtime = pdf_path.stat().st_mtime
|
||||
png_mtime = png_path.stat().st_mtime
|
||||
if png_mtime >= (pdf_mtime + 30):
|
||||
return
|
||||
else:
|
||||
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
|
||||
|
||||
print(f"Converting\n\t{pdf_path}\n → {png_path}")
|
||||
subprocess.run([
|
||||
"convert",
|
||||
"-density", str(CONFIG.png_dpi),
|
||||
str(pdf_path),
|
||||
"-quality", "95",
|
||||
str(png_path)
|
||||
], check=True)
|
||||
|
||||
|
||||
def needs_update(source_file: Path, target_file: Path) -> bool:
|
||||
"""
|
||||
Check if target file needs updating based on source modification time.
|
||||
|
||||
Args:
|
||||
source_file: Path to the source file
|
||||
target_file: Path to the target file
|
||||
|
||||
Returns:
|
||||
True if target needs update, False otherwise
|
||||
"""
|
||||
if not target_file.exists():
|
||||
return True
|
||||
|
||||
source_mtime = source_file.stat().st_mtime
|
||||
target_mtime = target_file.stat().st_mtime
|
||||
|
||||
return source_mtime > (target_mtime + 30)
|
||||
|
||||
|
||||
def build_gallery(source_dir: Path, web_dir: Path,
|
||||
relative_path: Path = None,
|
||||
inherited_metadata: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""
|
||||
Recursively build gallery structure from source directory.
|
||||
|
||||
Processes all PDF files in the source directory, converts them to PNG,
|
||||
copies both to the web directory, and generates index.html files with
|
||||
navigation and thumbnails. Now includes metadata support.
|
||||
|
||||
Args:
|
||||
source_dir: Source directory containing PDF files
|
||||
web_dir: Target web directory for gallery output
|
||||
relative_path: Relative path from gallery root (for navigation)
|
||||
inherited_metadata: Metadata inherited from parent directories
|
||||
"""
|
||||
if relative_path is None:
|
||||
relative_path = Path(".")
|
||||
|
||||
if inherited_metadata is None:
|
||||
inherited_metadata = {}
|
||||
|
||||
# Load folder-level metadata and merge with inherited metadata
|
||||
folder_metadata = load_folder_metadata(source_dir)
|
||||
current_metadata = merge_metadata(inherited_metadata, folder_metadata)
|
||||
|
||||
pdf_files = list(source_dir.glob("*.pdf"))
|
||||
|
||||
subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
|
||||
|
||||
items = []
|
||||
plot_metadata_cache = {}
|
||||
|
||||
for pdf_file in pdf_files:
|
||||
png_file = pdf_file.with_suffix(".png")
|
||||
|
||||
web_pdf = web_dir / pdf_file.name
|
||||
web_png = web_dir / png_file.name
|
||||
|
||||
if needs_update(pdf_file, web_pdf):
|
||||
print(f"Copying\n\t{pdf_file}\n → {web_pdf}")
|
||||
shutil.copy2(pdf_file, web_pdf)
|
||||
else:
|
||||
print(f"Skipping {pdf_file.name} (up to date)")
|
||||
|
||||
if not png_file.exists():
|
||||
convert_pdf_to_png(pdf_file)
|
||||
|
||||
if needs_update(png_file, web_png):
|
||||
print(f"Copying {png_file} to {web_png}")
|
||||
shutil.copy2(png_file, web_png)
|
||||
else:
|
||||
print(f"Skipping {png_file.name} (up to date)")
|
||||
|
||||
# Resolve metadata for this specific plot
|
||||
plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
|
||||
plot_metadata_cache[pdf_file.stem] = plot_metadata
|
||||
|
||||
# Get source file creation time (in seconds since epoch)
|
||||
source_creation_time = int(pdf_file.stat().st_ctime)
|
||||
|
||||
items.append({
|
||||
"name": pdf_file.stem,
|
||||
"pdf_href": pdf_file.name,
|
||||
"png_href": png_file.name,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
})
|
||||
|
||||
# Save metadata cache for this directory
|
||||
save_metadata_cache(web_dir, plot_metadata_cache)
|
||||
|
||||
subdir_names = []
|
||||
for subdir in subdirs:
|
||||
subdir_web = web_dir / subdir.name
|
||||
subdir_web.mkdir(exist_ok=True)
|
||||
subdir_relative = relative_path / subdir.name
|
||||
# Pass current metadata to subdirectories
|
||||
build_gallery(subdir, subdir_web, subdir_relative, current_metadata)
|
||||
subdir_names.append(subdir.name)
|
||||
|
||||
output_html = web_dir / "index.html"
|
||||
|
||||
# Always regenerate HTML to ensure subdirectory changes are reflected
|
||||
# This ensures new subdirectories appear in navigation
|
||||
force_regeneration = True
|
||||
if output_html.exists() and not force_regeneration:
|
||||
html_mtime = output_html.stat().st_mtime
|
||||
# Check if any subdirectory is newer than the HTML file
|
||||
for subdir in subdirs:
|
||||
if subdir.stat().st_mtime > html_mtime:
|
||||
print(f"Subdirectory {subdir.name} is newer, forcing regeneration")
|
||||
break
|
||||
|
||||
if relative_path == Path("."):
|
||||
title = "Gallery"
|
||||
else:
|
||||
title = f"Gallery: {relative_path}"
|
||||
|
||||
# Calculate statistics for current directory
|
||||
current_stats = calculate_directory_stats(web_dir)
|
||||
stats = {
|
||||
"file_count": len(items),
|
||||
"folder_count": len(subdir_names),
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets based on directory depth
|
||||
if relative_path == Path("."):
|
||||
assets_path = "../assets"
|
||||
else:
|
||||
# Count directory levels to go back to gallery, then to public_html
|
||||
depth = len(relative_path.parts)
|
||||
assets_path = "../" * (depth + 1) + "assets"
|
||||
|
||||
with output_html.open("w") as f:
|
||||
rendered_html = template.render(
|
||||
title=title,
|
||||
items=items,
|
||||
subdirs=subdir_names,
|
||||
relpath=str(relative_path),
|
||||
paths=CONFIG.paths,
|
||||
ui=CONFIG.ui,
|
||||
stats=stats,
|
||||
folder_metadata=current_metadata,
|
||||
assets_path=assets_path,
|
||||
source_dir=str(source_dir),
|
||||
metadata_file_path=get_metadata_file_path(source_dir)
|
||||
)
|
||||
f.write(rendered_html)
|
||||
|
||||
print(f"Generated {output_html}")
|
||||
|
||||
|
||||
def calculate_directory_stats(directory: Path) -> dict:
|
||||
"""
|
||||
Calculate statistics for a directory.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing file count, folder count, and total size
|
||||
"""
|
||||
stats = {
|
||||
"file_count": 0,
|
||||
"folder_count": 0,
|
||||
"total_size": 0,
|
||||
"pdf_size": 0,
|
||||
"png_size": 0
|
||||
}
|
||||
|
||||
if not directory.exists():
|
||||
return stats
|
||||
|
||||
for item in directory.rglob("*"):
|
||||
if item.is_file():
|
||||
stats["file_count"] += 1
|
||||
size = item.stat().st_size
|
||||
stats["total_size"] += size
|
||||
|
||||
if item.suffix.lower() == '.pdf':
|
||||
stats["pdf_size"] += size
|
||||
elif item.suffix.lower() == '.png':
|
||||
stats["png_size"] += size
|
||||
elif item.is_dir():
|
||||
stats["folder_count"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
Format file size in human readable format.
|
||||
|
||||
Args:
|
||||
size_bytes: Size in bytes
|
||||
|
||||
Returns:
|
||||
Formatted size string
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB"]
|
||||
size = float(size_bytes)
|
||||
i = 0
|
||||
while size >= 1024 and i < len(size_names) - 1:
|
||||
size /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{size:.1f} {size_names[i]}"
|
||||
|
||||
|
||||
def refresh_gallery_cgi():
|
||||
"""
|
||||
CGI handler to refresh the gallery from a web request.
|
||||
Outputs a minimal HTTP response and triggers gallery regeneration.
|
||||
"""
|
||||
import traceback
|
||||
print("Content-Type: text/plain\n")
|
||||
try:
|
||||
main()
|
||||
print("Gallery refreshed successfully.")
|
||||
except Exception as e:
|
||||
print(f"Error refreshing gallery: {e}")
|
||||
traceback.print_exc(file=sys.stdout)
|
||||
|
||||
|
||||
def main(clean_first: bool = False) -> None:
|
||||
"""
|
||||
Main entry point for gallery generation.
|
||||
|
||||
Args:
|
||||
clean_first: If True, removes and recreates the gallery directory
|
||||
|
||||
Processes all configured sources and generates the complete gallery
|
||||
structure in the web directory. Ensures assets are available.
|
||||
"""
|
||||
gallery_root = Path(CONFIG.web_folder) / CONFIG.plot_root
|
||||
|
||||
if clean_first and gallery_root.exists():
|
||||
print(f"Cleaning gallery directory {gallery_root}...")
|
||||
shutil.rmtree(gallery_root)
|
||||
|
||||
# Ensure gallery root exists
|
||||
gallery_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Always ensure assets are up to date
|
||||
assets_src = Path("assets")
|
||||
assets_dst = gallery_root.parent / "assets"
|
||||
|
||||
if assets_src.exists():
|
||||
# Update assets if they don't exist or are outdated
|
||||
main_css_src = assets_src / "css" / "main.css"
|
||||
main_css_dst = assets_dst / "css" / "main.css"
|
||||
if not assets_dst.exists() or needs_update(main_css_src, main_css_dst):
|
||||
if assets_dst.exists():
|
||||
shutil.rmtree(assets_dst)
|
||||
shutil.copytree(assets_src, assets_dst)
|
||||
print(f"Updated assets from {assets_src} to {assets_dst}")
|
||||
else:
|
||||
print(f"Warning: Assets directory {assets_src} not found")
|
||||
|
||||
for source in CONFIG.sources:
|
||||
source_path = Path(source.path)
|
||||
|
||||
if source_path.is_file() and source_path.suffix == '.pdf':
|
||||
source_web_dir = gallery_root / source.name
|
||||
source_web_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pdf_name = source_path.name
|
||||
png_name = source_path.with_suffix('.png').name
|
||||
|
||||
web_pdf_path = source_web_dir / pdf_name
|
||||
web_png_path = source_web_dir / png_name
|
||||
source_png_path = source_path.with_suffix('.png')
|
||||
|
||||
if needs_update(source_path, web_pdf_path):
|
||||
print(f"Copying {source_path} to {web_pdf_path}")
|
||||
shutil.copy2(source_path, web_pdf_path)
|
||||
else:
|
||||
print(f"Skipping {source_path.name} (up to date)")
|
||||
|
||||
if not source_png_path.exists():
|
||||
convert_pdf_to_png(source_path)
|
||||
|
||||
if needs_update(source_png_path, web_png_path):
|
||||
print(f"Copying {source_png_path} to {web_png_path}")
|
||||
shutil.copy2(source_png_path, web_png_path)
|
||||
else:
|
||||
print(f"Skipping {source_png_path.name} (up to date)")
|
||||
|
||||
# Get source file creation time for single file
|
||||
source_creation_time = int(source_path.stat().st_ctime)
|
||||
|
||||
items = [{
|
||||
"name": source_path.stem,
|
||||
"pdf_href": pdf_name,
|
||||
"png_href": png_name,
|
||||
"creation_time": source_creation_time
|
||||
}]
|
||||
|
||||
# Calculate statistics for single file
|
||||
current_stats = calculate_directory_stats(source_web_dir)
|
||||
stats = {
|
||||
"file_count": 1,
|
||||
"folder_count": 0,
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets for single file
|
||||
# Single files are at depth 1 (gallery_root/source.name/index.html)
|
||||
assets_path = "../assets"
|
||||
|
||||
output_html = source_web_dir / "index.html"
|
||||
with output_html.open("w") as f:
|
||||
f.write(template.render(
|
||||
title=source.name,
|
||||
items=items,
|
||||
subdirs=[],
|
||||
relpath=source.name,
|
||||
paths=CONFIG.paths,
|
||||
ui=CONFIG.ui,
|
||||
stats=stats,
|
||||
folder_metadata={},
|
||||
assets_path=assets_path,
|
||||
source_dir=str(source_path.parent),
|
||||
metadata_file_path=get_metadata_file_path(source_path.parent)
|
||||
))
|
||||
|
||||
print(f"Generated {output_html}")
|
||||
print(f"Processed {source.name}: {source.path}")
|
||||
|
||||
elif source_path.is_dir():
|
||||
source_web_dir = gallery_root / source.name
|
||||
source_web_dir.mkdir(parents=True, exist_ok=True)
|
||||
build_gallery(source_path, source_web_dir, Path(source.name))
|
||||
print(f"Processed {source.name}: {source.path}")
|
||||
else:
|
||||
print(f"Warning: Source {source.path} is neither a "
|
||||
f"directory nor a PDF file")
|
||||
|
||||
print("Done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if 'GATEWAY_INTERFACE' in os.environ:
|
||||
refresh_gallery_cgi()
|
||||
else:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Generate gallery')
|
||||
parser.add_argument(
|
||||
'--clean',
|
||||
action='store_true',
|
||||
help='Clean gallery directory before generation'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(clean_first=args.clean)
|
||||
Reference in New Issue
Block a user