Convert to proper python package
This commit is contained in:
Kylian Schmidt
@@ -0,0 +1 @@
|
||||
"""Utility modules for gallery generation."""
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Backup utilities for gallery."""
|
||||
|
||||
import zipfile
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def create_backup(
|
||||
web_folder: Path,
|
||||
backup_folder: Path
|
||||
) -> bool:
|
||||
"""
|
||||
Create a backup of the web folder.
|
||||
|
||||
Args:
|
||||
web_folder: Path to the web folder to backup
|
||||
backup_folder: Path to the backup directory
|
||||
|
||||
Returns:
|
||||
True if backup was created successfully, False otherwise
|
||||
"""
|
||||
try:
|
||||
today = datetime.date.today().strftime("%Y%m%d")
|
||||
backup_name = f"backup-{today}.zip"
|
||||
backup_path = backup_folder / backup_name
|
||||
|
||||
backup_folder.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if backup_path.exists():
|
||||
return True
|
||||
|
||||
with zipfile.ZipFile(backup_path, "w", zipfile.ZIP_DEFLATED) as zipf:
|
||||
for path in web_folder.rglob("*"):
|
||||
if path.is_file():
|
||||
arcname = path.relative_to(web_folder.parent)
|
||||
zipf.write(path, arcname)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not create backup: {e}")
|
||||
return False
|
||||
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
Metadata Management for Scientific Gallery Generator
|
||||
|
||||
This module handles loading, parsing, and caching of metadata for plots
|
||||
and folders in the gallery system. Supports YAML and JSON formats with
|
||||
hierarchical inheritance.
|
||||
|
||||
Features:
|
||||
- Load metadata from YAML/JSON files
|
||||
- Hierarchical metadata inheritance from parent folders
|
||||
- Plot-specific metadata overrides
|
||||
- Metadata caching for performance
|
||||
"""
|
||||
|
||||
import json
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Load metadata from a YAML or JSON file.
|
||||
|
||||
Args:
|
||||
metadata_path: Path to the metadata file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the metadata, empty dict if file doesn't exist
|
||||
or can't be parsed
|
||||
"""
|
||||
if not metadata_path.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
with metadata_path.open('r', encoding='utf-8') as f:
|
||||
suffix_lower = metadata_path.suffix.lower()
|
||||
if suffix_lower == '.yaml' or suffix_lower == '.yml':
|
||||
return yaml.safe_load(f) or {}
|
||||
elif metadata_path.suffix.lower() == '.json':
|
||||
return json.load(f) or {}
|
||||
else:
|
||||
print(f"Warning: Unknown metadata file format: "
|
||||
f"{metadata_path}")
|
||||
return {}
|
||||
except (yaml.YAMLError, json.JSONDecodeError, IOError) as e:
|
||||
print(f"Warning: Could not parse metadata file {metadata_path}: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def load_folder_metadata(folder_path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Load folder-level metadata from metadata.yaml, metadata.yml, or metadata.json.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the folder to check for metadata
|
||||
|
||||
Returns:
|
||||
Dictionary containing the folder metadata
|
||||
"""
|
||||
# Try YAML first, then JSON for backwards compatibility
|
||||
for filename in ['metadata.yaml', 'metadata.yml', 'metadata.json']:
|
||||
metadata_path = folder_path / filename
|
||||
if metadata_path.exists():
|
||||
return load_metadata_file(metadata_path)
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
def get_metadata_file_path(folder_path: Path) -> str:
|
||||
"""
|
||||
Get the metadata file path for a folder.
|
||||
|
||||
Returns existing file if found, otherwise suggests metadata.yaml.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the folder to check for metadata
|
||||
|
||||
Returns:
|
||||
String path to the metadata file (existing or suggested)
|
||||
"""
|
||||
# Preferred order: YAML first, then JSON
|
||||
preferred_files = ['metadata.yaml', 'metadata.yml', 'metadata.json']
|
||||
|
||||
for filename in preferred_files:
|
||||
metadata_path = folder_path / filename
|
||||
if metadata_path.exists():
|
||||
return str(metadata_path)
|
||||
|
||||
# If no file exists, suggest metadata.yaml (preferred format)
|
||||
return str(folder_path / 'metadata.yaml')
|
||||
|
||||
|
||||
def merge_metadata(
|
||||
parent_metadata: Dict[str, Any],
|
||||
child_metadata: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Merge parent and child metadata, with child values overriding parent.
|
||||
|
||||
Args:
|
||||
parent_metadata: Metadata from parent folder
|
||||
child_metadata: Metadata from current folder
|
||||
|
||||
Returns:
|
||||
Merged metadata dictionary
|
||||
"""
|
||||
merged = parent_metadata.copy()
|
||||
merged.update(child_metadata)
|
||||
return merged
|
||||
|
||||
|
||||
def resolve_metadata_for_plot(
|
||||
plot_path: Path,
|
||||
inherited_metadata: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Resolve metadata for a specific plot.
|
||||
|
||||
Merges inherited metadata with plot-specific metadata.
|
||||
|
||||
Args:
|
||||
plot_path: Path to the plot file (PDF)
|
||||
inherited_metadata: Metadata inherited from folder hierarchy
|
||||
|
||||
Returns:
|
||||
Final merged metadata for the plot
|
||||
"""
|
||||
plot_stem = plot_path.stem
|
||||
plot_dir = plot_path.parent
|
||||
|
||||
# Check for plot-specific metadata files
|
||||
for suffix in ['.yaml', '.yml', '.json']:
|
||||
plot_metadata_path = plot_dir / f"{plot_stem}{suffix}"
|
||||
if plot_metadata_path.exists():
|
||||
plot_metadata = load_metadata_file(plot_metadata_path)
|
||||
return merge_metadata(inherited_metadata, plot_metadata)
|
||||
|
||||
# No plot-specific metadata found, return inherited metadata
|
||||
return inherited_metadata.copy()
|
||||
|
||||
|
||||
def save_metadata_cache(
|
||||
web_dir: Path,
|
||||
plot_metadata_cache: Dict[str, Dict[str, Any]]
|
||||
) -> None:
|
||||
"""
|
||||
Save plot metadata cache to meta_cache.json in the web directory.
|
||||
|
||||
Args:
|
||||
web_dir: Web directory where the cache file should be saved
|
||||
plot_metadata_cache: Dictionary mapping plot names to their metadata
|
||||
"""
|
||||
cache_path = web_dir / "meta_cache.json"
|
||||
try:
|
||||
with cache_path.open('w', encoding='utf-8') as f:
|
||||
json.dump(plot_metadata_cache, f, indent=2, ensure_ascii=False)
|
||||
except IOError as e:
|
||||
print(f"Warning: Could not save metadata cache {cache_path}: {e}")
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Plot file processing and HTML rendering."""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
from gallery.utils.metadata import (
|
||||
resolve_metadata_for_plot,
|
||||
get_metadata_file_path,
|
||||
)
|
||||
from gallery.utils.stats import (
|
||||
calculate_directory_stats,
|
||||
format_file_size,
|
||||
)
|
||||
from gallery.config import GalleryConfig
|
||||
|
||||
|
||||
def process_html_file(
|
||||
html_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Process HTML plot file, copying it to web directory.
|
||||
|
||||
Args:
|
||||
html_file: Path to the source HTML file
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
web_html = web_dir / html_file.name
|
||||
|
||||
if needs_update(html_file, web_html):
|
||||
shutil.copy2(html_file, web_html)
|
||||
|
||||
# Get source file creation time
|
||||
source_creation_time = int(html_file.stat().st_ctime)
|
||||
|
||||
# Resolve metadata if provided
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(html_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": html_file.stem,
|
||||
"html_href": html_file.name,
|
||||
"is_html": True,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def process_plot_files(
|
||||
config: GalleryConfig,
|
||||
plot_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Process plot files (PDF/PNG or HTML), handling conversion and copying.
|
||||
|
||||
Args:
|
||||
config: Gallery configuration object
|
||||
plot_file: Path to the source plot file (PDF or HTML)
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
if plot_file.suffix.lower() == '.html':
|
||||
return process_html_file(plot_file, web_dir, current_metadata)
|
||||
|
||||
# Handle PDF files
|
||||
png_file = plot_file.with_suffix(".png")
|
||||
web_pdf = web_dir / plot_file.name
|
||||
web_png = web_dir / png_file.name
|
||||
|
||||
if needs_update(plot_file, web_pdf):
|
||||
shutil.copy2(plot_file, web_pdf)
|
||||
|
||||
if not png_file.exists():
|
||||
convert_pdf_to_png(plot_file, config=config)
|
||||
|
||||
if needs_update(png_file, web_png):
|
||||
shutil.copy2(png_file, web_png)
|
||||
|
||||
source_creation_time = int(plot_file.stat().st_ctime)
|
||||
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": plot_file.stem,
|
||||
"pdf_href": plot_file.name,
|
||||
"png_href": png_file.name,
|
||||
"is_html": False,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def render_gallery_page(
|
||||
config: GalleryConfig,
|
||||
template: Template,
|
||||
web_dir: Path,
|
||||
items: list,
|
||||
subdirs: list,
|
||||
relative_path: Path,
|
||||
title: str = None,
|
||||
metadata: dict = None
|
||||
) -> None:
|
||||
"""
|
||||
Unified template rendering for all gallery pages.
|
||||
|
||||
Args:
|
||||
config: Gallery configuration object
|
||||
template: Jinja2 template object
|
||||
web_dir: Target web directory
|
||||
items: List of plot items
|
||||
subdirs: List of subdirectory names
|
||||
relative_path: Relative path from gallery root
|
||||
title: Page title (optional)
|
||||
metadata: Metadata dictionary (optional)
|
||||
"""
|
||||
if title is None:
|
||||
title = "Gallery" if relative_path == Path(
|
||||
".") else f"Gallery: {relative_path}"
|
||||
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
# Calculate statistics
|
||||
current_stats = calculate_directory_stats(web_dir)
|
||||
stats = {
|
||||
"file_count": len(items),
|
||||
"folder_count": len(subdirs),
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets
|
||||
if relative_path == Path("."):
|
||||
assets_path = "../assets"
|
||||
else:
|
||||
depth = len(relative_path.parts)
|
||||
assets_path = "../" * (depth + 1) + "assets"
|
||||
|
||||
# For root level, show only directory structure
|
||||
if relative_path == Path("."):
|
||||
items = []
|
||||
|
||||
output_html = web_dir / "index.html"
|
||||
with output_html.open("w") as f:
|
||||
paths_dict = {
|
||||
"work_dir": str(Path.cwd()),
|
||||
"web_folder": str(config.web_folder),
|
||||
}
|
||||
ui_dict = {
|
||||
"max_recent_plots": 20,
|
||||
"search_debounce_ms": 300,
|
||||
}
|
||||
rendered_html = template.render(
|
||||
title=title,
|
||||
items=items,
|
||||
subdirs=subdirs,
|
||||
relpath=str(relative_path),
|
||||
paths=paths_dict,
|
||||
ui=ui_dict,
|
||||
stats=stats,
|
||||
folder_metadata=metadata,
|
||||
assets_path=assets_path,
|
||||
source_dir=str(web_dir),
|
||||
metadata_file_path=get_metadata_file_path(web_dir)
|
||||
)
|
||||
f.write(rendered_html)
|
||||
|
||||
|
||||
def convert_pdf_to_png(pdf_path: Path, config: GalleryConfig) -> None:
|
||||
"""
|
||||
Convert a PDF file to PNG format using ImageMagick.
|
||||
|
||||
Only converts if the PNG doesn't exist or if the PDF is newer than
|
||||
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
||||
|
||||
Args:
|
||||
pdf_path: Path to the source PDF file
|
||||
config: Gallery configuration object
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If ImageMagick conversion fails
|
||||
"""
|
||||
png_path = pdf_path.with_suffix(".png")
|
||||
|
||||
if png_path.exists():
|
||||
pdf_mtime = pdf_path.stat().st_mtime
|
||||
png_mtime = png_path.stat().st_mtime
|
||||
if png_mtime >= (pdf_mtime + 30):
|
||||
return
|
||||
|
||||
subprocess.run([
|
||||
"convert",
|
||||
"-density", str(config.png_dpi),
|
||||
str(pdf_path),
|
||||
"-quality", "95",
|
||||
str(png_path)
|
||||
], check=True)
|
||||
|
||||
|
||||
def needs_update(source_file: Path, target_file: Path) -> bool:
|
||||
"""
|
||||
Check if target file needs updating based on source modification time.
|
||||
|
||||
Args:
|
||||
source_file: Path to the source file
|
||||
target_file: Path to the target file
|
||||
|
||||
Returns:
|
||||
True if target needs update, False otherwise
|
||||
"""
|
||||
if not target_file.exists():
|
||||
return True
|
||||
|
||||
source_mtime = source_file.stat().st_mtime
|
||||
target_mtime = target_file.stat().st_mtime
|
||||
|
||||
return source_mtime > (target_mtime + 30)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Statistics calculation for gallery directories."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def calculate_directory_stats(directory: Path) -> dict:
|
||||
"""
|
||||
Calculate statistics for a directory.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing file count, folder count, and total size
|
||||
"""
|
||||
stats = {
|
||||
"file_count": 0,
|
||||
"folder_count": 0,
|
||||
"total_size": 0,
|
||||
"pdf_size": 0,
|
||||
"png_size": 0,
|
||||
}
|
||||
|
||||
if not directory.exists():
|
||||
return stats
|
||||
|
||||
for item in directory.rglob("*"):
|
||||
if item.is_file():
|
||||
stats["file_count"] += 1
|
||||
size = item.stat().st_size
|
||||
stats["total_size"] += size
|
||||
|
||||
if item.suffix.lower() == '.pdf':
|
||||
stats["pdf_size"] += size
|
||||
elif item.suffix.lower() == '.png':
|
||||
stats["png_size"] += size
|
||||
elif item.is_dir():
|
||||
stats["folder_count"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
Format file size in human readable format.
|
||||
|
||||
Args:
|
||||
size_bytes: Size in bytes
|
||||
|
||||
Returns:
|
||||
Formatted size string
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB"]
|
||||
size = float(size_bytes)
|
||||
i = 0
|
||||
while size >= 1024 and i < len(size_names) - 1:
|
||||
size /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{size:.1f} {size_names[i]}"
|
||||
Reference in New Issue
Block a user