move processing to dedicated file
This commit is contained in:
Kylian Schmidt
@@ -43,5 +43,7 @@ sources:
|
||||
path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/"
|
||||
- name: "needle_benchmarks"
|
||||
path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots"
|
||||
- name: "needle_fair_universe"
|
||||
path: "/ceph/kschmidt/needle/plots/fair_universe/"
|
||||
- name: "aido_convergence_study"
|
||||
path: "/work/kschmidt/aido/results_convergence/plots/"
|
||||
|
||||
+22
-272
@@ -13,7 +13,6 @@ Features:
|
||||
- Responsive grid layout with search and navigation
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
@@ -24,9 +23,12 @@ from utils.config import Config
|
||||
from utils.metadata import (
|
||||
load_folder_metadata,
|
||||
merge_metadata,
|
||||
resolve_metadata_for_plot,
|
||||
save_metadata_cache,
|
||||
get_metadata_file_path
|
||||
)
|
||||
from utils.processing import (
|
||||
process_plot_files,
|
||||
needs_update,
|
||||
render_gallery_page,
|
||||
)
|
||||
|
||||
|
||||
@@ -49,213 +51,6 @@ env.filters['strftime'] = strftime_filter
|
||||
template = env.get_template("templates/gallery.html")
|
||||
|
||||
|
||||
def process_html_file(html_file: Path, web_dir: Path, current_metadata: Dict[str, Any] = None) -> dict:
|
||||
"""
|
||||
Process HTML plot file, copying it to web directory.
|
||||
|
||||
Args:
|
||||
html_file: Path to the source HTML file
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
web_html = web_dir / html_file.name
|
||||
|
||||
if needs_update(html_file, web_html):
|
||||
shutil.copy2(html_file, web_html)
|
||||
else:
|
||||
print(f"Skipping {html_file.name} (up to date)")
|
||||
|
||||
# Get source file creation time
|
||||
source_creation_time = int(html_file.stat().st_ctime)
|
||||
|
||||
# Resolve metadata if provided
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(html_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": html_file.stem,
|
||||
"html_href": html_file.name,
|
||||
"is_html": True,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
def process_plot_files(
|
||||
plot_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Process plot files (PDF/PNG or HTML), handling conversion and copying.
|
||||
|
||||
Args:
|
||||
plot_file: Path to the source plot file (PDF or HTML)
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
if plot_file.suffix.lower() == '.html':
|
||||
return process_html_file(plot_file, web_dir, current_metadata)
|
||||
|
||||
# Handle PDF files
|
||||
png_file = plot_file.with_suffix(".png")
|
||||
web_pdf = web_dir / plot_file.name
|
||||
web_png = web_dir / png_file.name
|
||||
|
||||
if needs_update(plot_file, web_pdf):
|
||||
shutil.copy2(plot_file, web_pdf)
|
||||
else:
|
||||
print(f"Skipping {plot_file.name} (up to date)")
|
||||
|
||||
if not png_file.exists():
|
||||
convert_pdf_to_png(plot_file)
|
||||
|
||||
if needs_update(png_file, web_png):
|
||||
shutil.copy2(png_file, web_png)
|
||||
else:
|
||||
print(f"Skipping {png_file.name} (up to date)")
|
||||
|
||||
source_creation_time = int(plot_file.stat().st_ctime)
|
||||
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": plot_file.stem,
|
||||
"pdf_href": plot_file.name,
|
||||
"png_href": png_file.name,
|
||||
"is_html": False,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def render_gallery_page(
|
||||
web_dir: Path,
|
||||
items: list,
|
||||
subdirs: list,
|
||||
relative_path: Path,
|
||||
title: str = None,
|
||||
metadata: dict = None
|
||||
) -> None:
|
||||
"""
|
||||
Unified template rendering for all gallery pages.
|
||||
|
||||
Args:
|
||||
web_dir: Target web directory
|
||||
items: List of plot items
|
||||
subdirs: List of subdirectory names
|
||||
relative_path: Relative path from gallery root
|
||||
title: Page title (optional)
|
||||
metadata: Metadata dictionary (optional)
|
||||
"""
|
||||
if title is None:
|
||||
title = "Gallery" if relative_path == Path(
|
||||
".") else f"Gallery: {relative_path}"
|
||||
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
# Calculate statistics
|
||||
current_stats = calculate_directory_stats(web_dir)
|
||||
stats = {
|
||||
"file_count": len(items),
|
||||
"folder_count": len(subdirs),
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets
|
||||
if relative_path == Path("."):
|
||||
assets_path = "../assets"
|
||||
else:
|
||||
depth = len(relative_path.parts)
|
||||
assets_path = "../" * (depth + 1) + "assets"
|
||||
|
||||
# For root level, show only directory structure
|
||||
if relative_path == Path("."):
|
||||
items = []
|
||||
|
||||
output_html = web_dir / "index.html"
|
||||
with output_html.open("w") as f:
|
||||
rendered_html = template.render(
|
||||
title=title,
|
||||
items=items,
|
||||
subdirs=subdirs,
|
||||
relpath=str(relative_path),
|
||||
paths=CONFIG.paths,
|
||||
ui=CONFIG.ui,
|
||||
stats=stats,
|
||||
folder_metadata=metadata,
|
||||
assets_path=assets_path,
|
||||
source_dir=str(web_dir),
|
||||
metadata_file_path=get_metadata_file_path(web_dir)
|
||||
)
|
||||
f.write(rendered_html)
|
||||
|
||||
print(f"Generated {output_html}")
|
||||
|
||||
|
||||
def convert_pdf_to_png(pdf_path: Path) -> None:
|
||||
"""
|
||||
Convert a PDF file to PNG format using ImageMagick.
|
||||
|
||||
Only converts if the PNG doesn't exist or if the PDF is newer than
|
||||
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
||||
|
||||
Args:
|
||||
pdf_path: Path to the source PDF file
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If ImageMagick conversion fails
|
||||
"""
|
||||
png_path = pdf_path.with_suffix(".png")
|
||||
|
||||
if png_path.exists():
|
||||
pdf_mtime = pdf_path.stat().st_mtime
|
||||
png_mtime = png_path.stat().st_mtime
|
||||
if png_mtime >= (pdf_mtime + 30):
|
||||
return
|
||||
else:
|
||||
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
|
||||
|
||||
print(f"Converting {pdf_path} to PNG")
|
||||
subprocess.run([
|
||||
"convert",
|
||||
"-density", str(CONFIG.png_dpi),
|
||||
str(pdf_path),
|
||||
"-quality", "95",
|
||||
str(png_path)
|
||||
], check=True)
|
||||
|
||||
|
||||
def needs_update(source_file: Path, target_file: Path) -> bool:
|
||||
"""
|
||||
Check if target file needs updating based on source modification time.
|
||||
|
||||
Args:
|
||||
source_file: Path to the source file
|
||||
target_file: Path to the target file
|
||||
|
||||
Returns:
|
||||
True if target needs update, False otherwise
|
||||
"""
|
||||
if not target_file.exists():
|
||||
return True
|
||||
|
||||
source_mtime = source_file.stat().st_mtime
|
||||
target_mtime = target_file.stat().st_mtime
|
||||
|
||||
return source_mtime > (target_mtime + 30)
|
||||
|
||||
|
||||
def build_gallery(
|
||||
source_dir: Path,
|
||||
web_dir: Path,
|
||||
@@ -294,7 +89,12 @@ def build_gallery(
|
||||
|
||||
# Process all plot files (PDFs and HTMLs)
|
||||
for plot_file in plot_files:
|
||||
item = process_plot_files(plot_file, web_dir, current_metadata)
|
||||
item = process_plot_files(
|
||||
CONFIG=CONFIG,
|
||||
plot_file=plot_file,
|
||||
web_dir=web_dir,
|
||||
current_metadata=current_metadata,
|
||||
)
|
||||
items.append(item)
|
||||
plot_metadata_cache[plot_file.stem] = item["metadata"]
|
||||
|
||||
@@ -310,6 +110,8 @@ def build_gallery(
|
||||
subdir_names.append(subdir.name)
|
||||
|
||||
render_gallery_page(
|
||||
CONFIG=CONFIG,
|
||||
template=template,
|
||||
web_dir=web_dir,
|
||||
items=items,
|
||||
subdirs=subdir_names,
|
||||
@@ -318,66 +120,6 @@ def build_gallery(
|
||||
)
|
||||
|
||||
|
||||
def calculate_directory_stats(directory: Path) -> dict:
|
||||
"""
|
||||
Calculate statistics for a directory.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing file count, folder count, and total size
|
||||
"""
|
||||
stats = {
|
||||
"file_count": 0,
|
||||
"folder_count": 0,
|
||||
"total_size": 0,
|
||||
"pdf_size": 0,
|
||||
"png_size": 0,
|
||||
}
|
||||
|
||||
if not directory.exists():
|
||||
return stats
|
||||
|
||||
for item in directory.rglob("*"):
|
||||
if item.is_file():
|
||||
stats["file_count"] += 1
|
||||
size = item.stat().st_size
|
||||
stats["total_size"] += size
|
||||
|
||||
if item.suffix.lower() == '.pdf':
|
||||
stats["pdf_size"] += size
|
||||
elif item.suffix.lower() == '.png':
|
||||
stats["png_size"] += size
|
||||
elif item.is_dir():
|
||||
stats["folder_count"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
Format file size in human readable format.
|
||||
|
||||
Args:
|
||||
size_bytes: Size in bytes
|
||||
|
||||
Returns:
|
||||
Formatted size string
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB"]
|
||||
size = float(size_bytes)
|
||||
i = 0
|
||||
while size >= 1024 and i < len(size_names) - 1:
|
||||
size /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{size:.1f} {size_names[i]}"
|
||||
|
||||
|
||||
def main(clean_first: bool = False) -> None:
|
||||
"""
|
||||
Main entry point for gallery generation.
|
||||
@@ -418,8 +160,14 @@ def main(clean_first: bool = False) -> None:
|
||||
source_subdirs.append(source.name)
|
||||
|
||||
if source_path.is_file() and source_path.suffix == '.pdf':
|
||||
item = process_plot_files(source_path, source_web_dir)
|
||||
item = process_plot_files(
|
||||
CONFIG=CONFIG,
|
||||
plot_file=source_path,
|
||||
web_dir=source_web_dir,
|
||||
)
|
||||
render_gallery_page(
|
||||
CONFIG=CONFIG,
|
||||
template=template,
|
||||
web_dir=source_web_dir,
|
||||
items=[item],
|
||||
subdirs=[],
|
||||
@@ -436,6 +184,8 @@ def main(clean_first: bool = False) -> None:
|
||||
print(f"Processed {source.name}: {source.path}")
|
||||
|
||||
render_gallery_page(
|
||||
CONFIG=CONFIG,
|
||||
template=template,
|
||||
web_dir=gallery_root,
|
||||
items=[],
|
||||
subdirs=source_subdirs,
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import shutil
|
||||
from typing import Any, Dict
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from jinja2 import Template
|
||||
|
||||
from utils.metadata import (
|
||||
resolve_metadata_for_plot,
|
||||
get_metadata_file_path,
|
||||
)
|
||||
from utils.stats import (
|
||||
calculate_directory_stats,
|
||||
format_file_size,
|
||||
)
|
||||
from utils.config import Config
|
||||
|
||||
|
||||
def process_html_file(
|
||||
html_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Process HTML plot file, copying it to web directory.
|
||||
|
||||
Args:
|
||||
html_file: Path to the source HTML file
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
web_html = web_dir / html_file.name
|
||||
|
||||
if needs_update(html_file, web_html):
|
||||
shutil.copy2(html_file, web_html)
|
||||
else:
|
||||
print(f"Skipping {html_file.name} (up to date)")
|
||||
|
||||
# Get source file creation time
|
||||
source_creation_time = int(html_file.stat().st_ctime)
|
||||
|
||||
# Resolve metadata if provided
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(html_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": html_file.stem,
|
||||
"html_href": html_file.name,
|
||||
"is_html": True,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def process_plot_files(
|
||||
CONFIG: Config,
|
||||
plot_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Process plot files (PDF/PNG or HTML), handling conversion and copying.
|
||||
|
||||
Args:
|
||||
plot_file: Path to the source plot file (PDF or HTML)
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
if plot_file.suffix.lower() == '.html':
|
||||
return process_html_file(plot_file, web_dir, current_metadata)
|
||||
|
||||
# Handle PDF files
|
||||
png_file = plot_file.with_suffix(".png")
|
||||
web_pdf = web_dir / plot_file.name
|
||||
web_png = web_dir / png_file.name
|
||||
|
||||
if needs_update(plot_file, web_pdf):
|
||||
shutil.copy2(plot_file, web_pdf)
|
||||
else:
|
||||
print(f"Skipping {plot_file.name} (up to date)")
|
||||
|
||||
if not png_file.exists():
|
||||
convert_pdf_to_png(plot_file, CONFIG=CONFIG)
|
||||
|
||||
if needs_update(png_file, web_png):
|
||||
shutil.copy2(png_file, web_png)
|
||||
else:
|
||||
print(f"Skipping {png_file.name} (up to date)")
|
||||
|
||||
source_creation_time = int(plot_file.stat().st_ctime)
|
||||
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": plot_file.stem,
|
||||
"pdf_href": plot_file.name,
|
||||
"png_href": png_file.name,
|
||||
"is_html": False,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def render_gallery_page(
|
||||
CONFIG: Config,
|
||||
template: Template,
|
||||
web_dir: Path,
|
||||
items: list,
|
||||
subdirs: list,
|
||||
relative_path: Path,
|
||||
title: str = None,
|
||||
metadata: dict = None
|
||||
) -> None:
|
||||
"""
|
||||
Unified template rendering for all gallery pages.
|
||||
|
||||
Args:
|
||||
web_dir: Target web directory
|
||||
items: List of plot items
|
||||
subdirs: List of subdirectory names
|
||||
relative_path: Relative path from gallery root
|
||||
title: Page title (optional)
|
||||
metadata: Metadata dictionary (optional)
|
||||
"""
|
||||
if title is None:
|
||||
title = "Gallery" if relative_path == Path(
|
||||
".") else f"Gallery: {relative_path}"
|
||||
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
# Calculate statistics
|
||||
current_stats = calculate_directory_stats(web_dir)
|
||||
stats = {
|
||||
"file_count": len(items),
|
||||
"folder_count": len(subdirs),
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets
|
||||
if relative_path == Path("."):
|
||||
assets_path = "../assets"
|
||||
else:
|
||||
depth = len(relative_path.parts)
|
||||
assets_path = "../" * (depth + 1) + "assets"
|
||||
|
||||
# For root level, show only directory structure
|
||||
if relative_path == Path("."):
|
||||
items = []
|
||||
|
||||
output_html = web_dir / "index.html"
|
||||
with output_html.open("w") as f:
|
||||
rendered_html = template.render(
|
||||
title=title,
|
||||
items=items,
|
||||
subdirs=subdirs,
|
||||
relpath=str(relative_path),
|
||||
paths=CONFIG.paths,
|
||||
ui=CONFIG.ui,
|
||||
stats=stats,
|
||||
folder_metadata=metadata,
|
||||
assets_path=assets_path,
|
||||
source_dir=str(web_dir),
|
||||
metadata_file_path=get_metadata_file_path(web_dir)
|
||||
)
|
||||
f.write(rendered_html)
|
||||
|
||||
print(f"Generated {output_html}")
|
||||
|
||||
|
||||
def convert_pdf_to_png(pdf_path: Path, CONFIG: Config) -> None:
|
||||
"""
|
||||
Convert a PDF file to PNG format using ImageMagick.
|
||||
|
||||
Only converts if the PNG doesn't exist or if the PDF is newer than
|
||||
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
||||
|
||||
Args:
|
||||
pdf_path: Path to the source PDF file
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If ImageMagick conversion fails
|
||||
"""
|
||||
png_path = pdf_path.with_suffix(".png")
|
||||
|
||||
if png_path.exists():
|
||||
pdf_mtime = pdf_path.stat().st_mtime
|
||||
png_mtime = png_path.stat().st_mtime
|
||||
if png_mtime >= (pdf_mtime + 30):
|
||||
return
|
||||
else:
|
||||
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
|
||||
|
||||
print(f"Converting {pdf_path} to PNG")
|
||||
subprocess.run([
|
||||
"convert",
|
||||
"-density", str(CONFIG.png_dpi),
|
||||
str(pdf_path),
|
||||
"-quality", "95",
|
||||
str(png_path)
|
||||
], check=True)
|
||||
|
||||
|
||||
def needs_update(source_file: Path, target_file: Path) -> bool:
|
||||
"""
|
||||
Check if target file needs updating based on source modification time.
|
||||
|
||||
Args:
|
||||
source_file: Path to the source file
|
||||
target_file: Path to the target file
|
||||
|
||||
Returns:
|
||||
True if target needs update, False otherwise
|
||||
"""
|
||||
if not target_file.exists():
|
||||
return True
|
||||
|
||||
source_mtime = source_file.stat().st_mtime
|
||||
target_mtime = target_file.stat().st_mtime
|
||||
|
||||
return source_mtime > (target_mtime + 30)
|
||||
@@ -0,0 +1,62 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def calculate_directory_stats(directory: Path) -> dict:
|
||||
"""
|
||||
Calculate statistics for a directory.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing file count, folder count, and total size
|
||||
"""
|
||||
stats = {
|
||||
"file_count": 0,
|
||||
"folder_count": 0,
|
||||
"total_size": 0,
|
||||
"pdf_size": 0,
|
||||
"png_size": 0,
|
||||
}
|
||||
|
||||
if not directory.exists():
|
||||
return stats
|
||||
|
||||
for item in directory.rglob("*"):
|
||||
if item.is_file():
|
||||
stats["file_count"] += 1
|
||||
size = item.stat().st_size
|
||||
stats["total_size"] += size
|
||||
|
||||
if item.suffix.lower() == '.pdf':
|
||||
stats["pdf_size"] += size
|
||||
elif item.suffix.lower() == '.png':
|
||||
stats["png_size"] += size
|
||||
elif item.is_dir():
|
||||
stats["folder_count"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
Format file size in human readable format.
|
||||
|
||||
Args:
|
||||
size_bytes: Size in bytes
|
||||
|
||||
Returns:
|
||||
Formatted size string
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB"]
|
||||
size = float(size_bytes)
|
||||
i = 0
|
||||
while size >= 1024 and i < len(size_names) - 1:
|
||||
size /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{size:.1f} {size_names[i]}"
|
||||
|
||||
Reference in New Issue
Block a user