lars/ETPlot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix missing top-level tree index.html. Clean up duplicates in main script

Browse Source
This commit is contained in:
Kylian Schmidt
2025-09-26 08:53:37 +02:00
parent 425685874a
commit 041c761df0
2 changed files with 164 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.yaml
+2
View File
@@ -43,3 +43,5 @@ sources:
path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/"
- name: "needle_benchmarks"
path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots"
- name: "aido_convergence_study"
path: "/work/kschmidt/aido/results_convergence/plots/"
generate_gallery.py
+157 -188
View File
@@ -15,7 +15,6 @@ Features:
import subprocess
import shutil
import sys
from pathlib import Path
from typing import Dict, Any, Optional
from datetime import datetime
@@ -34,12 +33,12 @@ from utils.metadata import (
CONFIG = Config.from_yaml("config.yaml")
def datetime_from_timestamp(timestamp):
def datetime_from_timestamp(timestamp: float) -> datetime:
"""Convert a Unix timestamp to a datetime object."""
return datetime.fromtimestamp(timestamp)
def strftime_filter(dt, fmt):
def strftime_filter(dt: datetime, fmt: str) -> str:
"""Format a datetime object using strftime."""
return dt.strftime(fmt)
@@ -50,6 +49,122 @@ env.filters['strftime'] = strftime_filter
template = env.get_template("templates/gallery.html")
def process_plot_files(
pdf_file: Path,
web_dir: Path,
current_metadata: Dict[str, Any] = None,
) -> dict:
"""
Process PDF and PNG files, handling conversion and copying.
Args:
pdf_file: Path to the source PDF file
web_dir: Target web directory
current_metadata: Current metadata dictionary for the plot
Returns:
Dictionary containing plot information
"""
png_file = pdf_file.with_suffix(".png")
web_pdf = web_dir / pdf_file.name
web_png = web_dir / png_file.name
if needs_update(pdf_file, web_pdf):
shutil.copy2(pdf_file, web_pdf)
else:
print(f"Skipping {pdf_file.name} (up to date)")
if not png_file.exists():
convert_pdf_to_png(pdf_file)
if needs_update(png_file, web_png):
shutil.copy2(png_file, web_png)
else:
print(f"Skipping {png_file.name} (up to date)")
# Get source file creation time
source_creation_time = int(pdf_file.stat().st_ctime)
# Resolve metadata if provided
plot_metadata = {}
if current_metadata is not None:
plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
return {
"name": pdf_file.stem,
"pdf_href": pdf_file.name,
"png_href": png_file.name,
"metadata": plot_metadata,
"creation_time": source_creation_time
}
def render_gallery_page(
web_dir: Path,
items: list,
subdirs: list,
relative_path: Path,
title: str = None,
metadata: dict = None
) -> None:
"""
Unified template rendering for all gallery pages.
Args:
web_dir: Target web directory
items: List of plot items
subdirs: List of subdirectory names
relative_path: Relative path from gallery root
title: Page title (optional)
metadata: Metadata dictionary (optional)
"""
if title is None:
title = "Gallery" if relative_path == Path(
".") else f"Gallery: {relative_path}"
if metadata is None:
metadata = {}
# Calculate statistics
current_stats = calculate_directory_stats(web_dir)
stats = {
"file_count": len(items),
"folder_count": len(subdirs),
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
}
# Calculate relative path to assets
if relative_path == Path("."):
assets_path = "../assets"
else:
depth = len(relative_path.parts)
assets_path = "../" * (depth + 1) + "assets"
# For root level, show only directory structure
if relative_path == Path("."):
items = []
output_html = web_dir / "index.html"
with output_html.open("w") as f:
rendered_html = template.render(
title=title,
items=items,
subdirs=subdirs,
relpath=str(relative_path),
paths=CONFIG.paths,
ui=CONFIG.ui,
stats=stats,
folder_metadata=metadata,
assets_path=assets_path,
source_dir=str(web_dir),
metadata_file_path=get_metadata_file_path(web_dir)
)
f.write(rendered_html)
print(f"Generated {output_html}")
def convert_pdf_to_png(pdf_path: Path) -> None:
"""
Convert a PDF file to PNG format using ImageMagick.
@@ -73,7 +188,7 @@ def convert_pdf_to_png(pdf_path: Path) -> None:
else:
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
print(f"Converting\n\t{pdf_path}\n → {png_path}")
print(f"Converting {pdf_path} to PNG")
subprocess.run([
"convert",
"-density", str(CONFIG.png_dpi),
@@ -103,9 +218,12 @@ def needs_update(source_file: Path, target_file: Path) -> bool:
return source_mtime > (target_mtime + 30)
def build_gallery(source_dir: Path, web_dir: Path,
def build_gallery(
source_dir: Path,
web_dir: Path,
relative_path: Path = None,
inherited_metadata: Optional[Dict[str, Any]] = None) -> None:
inherited_metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""
Recursively build gallery structure from source directory.
@@ -125,117 +243,36 @@ def build_gallery(source_dir: Path, web_dir: Path,
if inherited_metadata is None:
inherited_metadata = {}
# Load folder-level metadata and merge with inherited metadata
folder_metadata = load_folder_metadata(source_dir)
current_metadata = merge_metadata(inherited_metadata, folder_metadata)
pdf_files = list(source_dir.glob("*.pdf"))
subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
items = []
plot_metadata_cache = {}
for pdf_file in pdf_files:
png_file = pdf_file.with_suffix(".png")
item = process_plot_files(pdf_file, web_dir, current_metadata)
items.append(item)
plot_metadata_cache[pdf_file.stem] = item["metadata"]
web_pdf = web_dir / pdf_file.name
web_png = web_dir / png_file.name
if needs_update(pdf_file, web_pdf):
print(f"Copying\n\t{pdf_file}\n → {web_pdf}")
shutil.copy2(pdf_file, web_pdf)
else:
print(f"Skipping {pdf_file.name} (up to date)")
if not png_file.exists():
convert_pdf_to_png(pdf_file)
if needs_update(png_file, web_png):
print(f"Copying {png_file} to {web_png}")
shutil.copy2(png_file, web_png)
else:
print(f"Skipping {png_file.name} (up to date)")
# Resolve metadata for this specific plot
plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
plot_metadata_cache[pdf_file.stem] = plot_metadata
# Get source file creation time (in seconds since epoch)
source_creation_time = int(pdf_file.stat().st_ctime)
items.append({
"name": pdf_file.stem,
"pdf_href": pdf_file.name,
"png_href": png_file.name,
"metadata": plot_metadata,
"creation_time": source_creation_time
})
# Save metadata cache for this directory
save_metadata_cache(web_dir, plot_metadata_cache)
subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
subdir_names = []
for subdir in subdirs:
subdir_web = web_dir / subdir.name
subdir_web.mkdir(exist_ok=True)
subdir_relative = relative_path / subdir.name
# Pass current metadata to subdirectories
build_gallery(subdir, subdir_web, subdir_relative, current_metadata)
subdir_names.append(subdir.name)
output_html = web_dir / "index.html"
# Always regenerate HTML to ensure subdirectory changes are reflected
# This ensures new subdirectories appear in navigation
force_regeneration = True
if output_html.exists() and not force_regeneration:
html_mtime = output_html.stat().st_mtime
# Check if any subdirectory is newer than the HTML file
for subdir in subdirs:
if subdir.stat().st_mtime > html_mtime:
print(f"Subdirectory {subdir.name} is newer, forcing regeneration")
break
if relative_path == Path("."):
title = "Gallery"
else:
title = f"Gallery: {relative_path}"
# Calculate statistics for current directory
current_stats = calculate_directory_stats(web_dir)
stats = {
"file_count": len(items),
"folder_count": len(subdir_names),
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
}
# Calculate relative path to assets based on directory depth
if relative_path == Path("."):
assets_path = "../assets"
else:
# Count directory levels to go back to gallery, then to public_html
depth = len(relative_path.parts)
assets_path = "../" * (depth + 1) + "assets"
with output_html.open("w") as f:
rendered_html = template.render(
title=title,
render_gallery_page(
web_dir=web_dir,
items=items,
subdirs=subdir_names,
relpath=str(relative_path),
paths=CONFIG.paths,
ui=CONFIG.ui,
stats=stats,
folder_metadata=current_metadata,
assets_path=assets_path,
source_dir=str(source_dir),
metadata_file_path=get_metadata_file_path(source_dir)
relative_path=relative_path,
metadata=current_metadata
)
f.write(rendered_html)
print(f"Generated {output_html}")
def calculate_directory_stats(directory: Path) -> dict:
@@ -253,7 +290,7 @@ def calculate_directory_stats(directory: Path) -> dict:
"folder_count": 0,
"total_size": 0,
"pdf_size": 0,
"png_size": 0
"png_size": 0,
}
if not directory.exists():
@@ -298,21 +335,6 @@ def format_file_size(size_bytes: int) -> str:
return f"{size:.1f} {size_names[i]}"
def refresh_gallery_cgi():
"""
CGI handler to refresh the gallery from a web request.
Outputs a minimal HTTP response and triggers gallery regeneration.
"""
import traceback
print("Content-Type: text/plain\n")
try:
main()
print("Gallery refreshed successfully.")
except Exception as e:
print(f"Error refreshing gallery: {e}")
traceback.print_exc(file=sys.stdout)
def main(clean_first: bool = False) -> None:
"""
Main entry point for gallery generation.
@@ -329,15 +351,11 @@ def main(clean_first: bool = False) -> None:
print(f"Cleaning gallery directory {gallery_root}...")
shutil.rmtree(gallery_root)
# Ensure gallery root exists
gallery_root.mkdir(parents=True, exist_ok=True)
# Always ensure assets are up to date
assets_src = Path("assets")
assets_dst = gallery_root.parent / "assets"
if assets_src.exists():
# Update assets if they don't exist or are outdated
main_css_src = assets_src / "css" / "main.css"
main_css_dst = assets_dst / "css" / "main.css"
if not assets_dst.exists() or needs_update(main_css_src, main_css_dst):
@@ -348,88 +366,39 @@ def main(clean_first: bool = False) -> None:
else:
print(f"Warning: Assets directory {assets_src} not found")
source_subdirs = []
for source in CONFIG.sources:
source_path = Path(source.path)
source_web_dir = gallery_root / source.name
source_web_dir.mkdir(parents=True, exist_ok=True)
source_subdirs.append(source.name)
if source_path.is_file() and source_path.suffix == '.pdf':
source_web_dir = gallery_root / source.name
source_web_dir.mkdir(parents=True, exist_ok=True)
pdf_name = source_path.name
png_name = source_path.with_suffix('.png').name
web_pdf_path = source_web_dir / pdf_name
web_png_path = source_web_dir / png_name
source_png_path = source_path.with_suffix('.png')
if needs_update(source_path, web_pdf_path):
print(f"Copying {source_path} to {web_pdf_path}")
shutil.copy2(source_path, web_pdf_path)
else:
print(f"Skipping {source_path.name} (up to date)")
if not source_png_path.exists():
convert_pdf_to_png(source_path)
if needs_update(source_png_path, web_png_path):
print(f"Copying {source_png_path} to {web_png_path}")
shutil.copy2(source_png_path, web_png_path)
else:
print(f"Skipping {source_png_path.name} (up to date)")
# Get source file creation time for single file
source_creation_time = int(source_path.stat().st_ctime)
items = [{
"name": source_path.stem,
"pdf_href": pdf_name,
"png_href": png_name,
"creation_time": source_creation_time
}]
# Calculate statistics for single file
current_stats = calculate_directory_stats(source_web_dir)
stats = {
"file_count": 1,
"folder_count": 0,
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
}
# Calculate relative path to assets for single file
# Single files are at depth 1 (gallery_root/source.name/index.html)
assets_path = "../assets"
output_html = source_web_dir / "index.html"
with output_html.open("w") as f:
f.write(template.render(
title=source.name,
items=items,
item = process_plot_files(source_path, source_web_dir)
render_gallery_page(
web_dir=source_web_dir,
items=[item],
subdirs=[],
relpath=source.name,
paths=CONFIG.paths,
ui=CONFIG.ui,
stats=stats,
folder_metadata={},
assets_path=assets_path,
source_dir=str(source_path.parent),
metadata_file_path=get_metadata_file_path(
source_path.parent)
))
print(f"Generated {output_html}")
print(f"Processed {source.name}: {source.path}")
relative_path=Path(source.name)
)
elif source_path.is_dir():
source_web_dir = gallery_root / source.name
source_web_dir.mkdir(parents=True, exist_ok=True)
build_gallery(source_path, source_web_dir, Path(source.name))
print(f"Processed {source.name}: {source.path}")
else:
print(f"Warning: Source {source.path} is neither a "
f"directory nor a PDF file")
print(
f"Warning: Source {source.path} is neither a "
f"directory nor a PDF file. Skipping."
)
print("Done")
print(f"Processed {source.name}: {source.path}")
render_gallery_page(
web_dir=gallery_root,
items=[],
subdirs=source_subdirs,
relative_path=Path("."),
title="Gallery Root"
)
if __name__ == "__main__":