3b9d1ef1a8
- Replace the Docker build/publish CI stages with ruff (lint + format check), ty (type check), pip-audit, and pytest run directly against python:3.11-slim; Docker remains for manual/server deployment only. - Swap black/pylint/mypy for ruff/ty across pyproject.toml, and fix every resulting lint, format, and type diagnostic in gallery/ and plotstyle/. - Fix tests broken/stale from before the package restructuring: wrong `utils.*` import paths, mock patch targets pointed at the wrong module, and PDF-conversion tests still assuming ImageMagick instead of the current PyMuPDF-first path. Drop test_container.py (obsolete Docker-container smoke tests, fully superseded elsewhere). - Add a plain-venv + systemd --user timer deployment path (deploy/systemd/) for HPC login nodes without a Docker daemon, where public_html is already served by existing infrastructure. - Document both in CLAUDE.md, including running CI's checks locally before committing. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
152 lines
4.6 KiB
Python
152 lines
4.6 KiB
Python
"""
|
|
Metadata Management for Scientific Gallery Generator
|
|
|
|
This module handles loading, parsing, and caching of metadata for plots
|
|
and folders in the gallery system. Supports YAML and JSON formats with
|
|
hierarchical inheritance.
|
|
|
|
Features:
|
|
- Load metadata from YAML/JSON files
|
|
- Hierarchical metadata inheritance from parent folders
|
|
- Plot-specific metadata overrides
|
|
- Metadata caching for performance
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Dict
|
|
|
|
import yaml
|
|
|
|
|
|
def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
|
|
"""
|
|
Load metadata from a YAML or JSON file.
|
|
|
|
Args:
|
|
metadata_path: Path to the metadata file
|
|
|
|
Returns:
|
|
Dictionary containing the metadata, empty dict if file doesn't exist
|
|
or can't be parsed
|
|
"""
|
|
if not metadata_path.exists():
|
|
return {}
|
|
|
|
try:
|
|
with metadata_path.open("r", encoding="utf-8") as f:
|
|
suffix_lower = metadata_path.suffix.lower()
|
|
if suffix_lower == ".yaml" or suffix_lower == ".yml":
|
|
return yaml.safe_load(f) or {}
|
|
elif metadata_path.suffix.lower() == ".json":
|
|
return json.load(f) or {}
|
|
else:
|
|
print(f"Warning: Unknown metadata file format: {metadata_path}")
|
|
return {}
|
|
except (yaml.YAMLError, json.JSONDecodeError, IOError) as e:
|
|
print(f"Warning: Could not parse metadata file {metadata_path}: {e}")
|
|
raise e
|
|
|
|
|
|
def load_folder_metadata(folder_path: Path) -> Dict[str, Any]:
|
|
"""
|
|
Load folder-level metadata from metadata.yaml, metadata.yml, or metadata.json.
|
|
|
|
Args:
|
|
folder_path: Path to the folder to check for metadata
|
|
|
|
Returns:
|
|
Dictionary containing the folder metadata
|
|
"""
|
|
# Try YAML first, then JSON for backwards compatibility
|
|
for filename in ["metadata.yaml", "metadata.yml", "metadata.json"]:
|
|
metadata_path = folder_path / filename
|
|
|
|
if metadata_path.exists():
|
|
return load_metadata_file(metadata_path)
|
|
|
|
return {}
|
|
|
|
|
|
def get_metadata_file_path(folder_path: Path) -> str:
|
|
"""
|
|
Get the metadata file path for a folder.
|
|
|
|
Returns existing file if found, otherwise suggests metadata.yaml.
|
|
|
|
Args:
|
|
folder_path: Path to the folder to check for metadata
|
|
|
|
Returns:
|
|
String path to the metadata file (existing or suggested)
|
|
"""
|
|
# Preferred order: YAML first, then JSON
|
|
preferred_files = ["metadata.yaml", "metadata.yml", "metadata.json"]
|
|
|
|
for filename in preferred_files:
|
|
metadata_path = folder_path / filename
|
|
if metadata_path.exists():
|
|
return str(metadata_path)
|
|
|
|
# If no file exists, suggest metadata.yaml (preferred format)
|
|
return str(folder_path / "metadata.yaml")
|
|
|
|
|
|
def merge_metadata(parent_metadata: Dict[str, Any], child_metadata: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Merge parent and child metadata, with child values overriding parent.
|
|
|
|
Args:
|
|
parent_metadata: Metadata from parent folder
|
|
child_metadata: Metadata from current folder
|
|
|
|
Returns:
|
|
Merged metadata dictionary
|
|
"""
|
|
merged = parent_metadata.copy()
|
|
merged.update(child_metadata)
|
|
return merged
|
|
|
|
|
|
def resolve_metadata_for_plot(plot_path: Path, inherited_metadata: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Resolve metadata for a specific plot.
|
|
|
|
Merges inherited metadata with plot-specific metadata.
|
|
|
|
Args:
|
|
plot_path: Path to the plot file (PDF)
|
|
inherited_metadata: Metadata inherited from folder hierarchy
|
|
|
|
Returns:
|
|
Final merged metadata for the plot
|
|
"""
|
|
plot_stem = plot_path.stem
|
|
plot_dir = plot_path.parent
|
|
|
|
# Check for plot-specific metadata files
|
|
for suffix in [".yaml", ".yml", ".json"]:
|
|
plot_metadata_path = plot_dir / f"{plot_stem}{suffix}"
|
|
if plot_metadata_path.exists():
|
|
plot_metadata = load_metadata_file(plot_metadata_path)
|
|
return merge_metadata(inherited_metadata, plot_metadata)
|
|
|
|
# No plot-specific metadata found, return inherited metadata
|
|
return inherited_metadata.copy()
|
|
|
|
|
|
def save_metadata_cache(web_dir: Path, plot_metadata_cache: Dict[str, Dict[str, Any]]) -> None:
|
|
"""
|
|
Save plot metadata cache to meta_cache.json in the web directory.
|
|
|
|
Args:
|
|
web_dir: Web directory where the cache file should be saved
|
|
plot_metadata_cache: Dictionary mapping plot names to their metadata
|
|
"""
|
|
cache_path = web_dir / "meta_cache.json"
|
|
try:
|
|
with cache_path.open("w", encoding="utf-8") as f:
|
|
json.dump(plot_metadata_cache, f, indent=2, ensure_ascii=False)
|
|
except IOError as e:
|
|
print(f"Warning: Could not save metadata cache {cache_path}: {e}")
|