lars/ETPlot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite CI to lint/typecheck/audit/test only; add HPC deployment path

Browse Source 
- Replace the Docker build/publish CI stages with ruff (lint + format
  check), ty (type check), pip-audit, and pytest run directly against
  python:3.11-slim; Docker remains for manual/server deployment only.
- Swap black/pylint/mypy for ruff/ty across pyproject.toml, and fix
  every resulting lint, format, and type diagnostic in gallery/ and
  plotstyle/.
- Fix tests broken/stale from before the package restructuring: wrong
  `utils.*` import paths, mock patch targets pointed at the wrong
  module, and PDF-conversion tests still assuming ImageMagick instead
  of the current PyMuPDF-first path. Drop test_container.py (obsolete
  Docker-container smoke tests, fully superseded elsewhere).
- Add a plain-venv + systemd --user timer deployment path
  (deploy/systemd/) for HPC login nodes without a Docker daemon, where
  public_html is already served by existing infrastructure.
- Document both in CLAUDE.md, including running CI's checks locally
  before committing.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Lars Bogner
2026-07-22 15:29:55 +02:00
parent 4ba321b327
commit 3b9d1ef1a8
27 changed files with 752 additions and 915 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gallery/__init__.py
+18 -19
View File
@@ -21,17 +21,23 @@ Example usage:
__version__ = "0.1.0"
__author__ = "K. Schmidt"
from gallery.api import generate
from gallery.builder import build_gallery, get_template
from gallery.config import (
GalleryConfig,
GallerySource,
GalleryDefaults,
GallerySource,
)
from gallery.api import generate
# Export utility functions for testing and advanced usage
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
from gallery.utils.datetime_utils import (
datetime_from_timestamp,
strftime_filter,
)
from gallery.utils.metadata import (
load_folder_metadata,
load_metadata_file,
merge_metadata,
resolve_metadata_for_plot,
save_metadata_cache,
)
from gallery.utils.processing import (
convert_pdf_to_png,
@@ -39,18 +45,12 @@ from gallery.utils.processing import (
process_plot_files,
render_gallery_page,
)
from gallery.utils.metadata import (
load_folder_metadata,
merge_metadata,
save_metadata_cache,
load_metadata_file,
resolve_metadata_for_plot,
# Export utility functions for testing and advanced usage
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
)
from gallery.utils.datetime_utils import (
datetime_from_timestamp,
strftime_filter,
)
from gallery.builder import build_gallery, get_template
__all__ = [
"generate",
@@ -74,4 +74,3 @@ __all__ = [
"datetime_from_timestamp",
"strftime_filter",
]
gallery/api.py
+27 -57
View File
@@ -6,19 +6,19 @@ Provides the primary entry point for programmatic gallery generation.
import shutil
from pathlib import Path
from typing import Union, List, Dict, Any
from typing import Any, Dict, List, Optional, Union, cast
from gallery.builder import build_gallery, copy_assets, get_template
from gallery.config import GalleryConfig, GallerySource
from gallery.builder import get_template, build_gallery, copy_assets
def generate(
config: Union[GalleryConfig, str, Path] = None,
web_folder: Union[str, Path] = None,
sources: List[Union[GallerySource, Dict[str, Any]]] = None,
config: Optional[Union[GalleryConfig, str, Path]] = None,
web_folder: Optional[Union[str, Path]] = None,
sources: Optional[List[Union[GallerySource, Dict[str, Any]]]] = None,
clean_first: bool = False,
verbose: bool = False,
source_to_update: GallerySource = None,
source_to_update: Optional[GallerySource] = None,
) -> bool:
"""
Generate a scientific gallery from plot sources.
@@ -79,20 +79,11 @@ def generate(
if isinstance(config, (str, Path)):
config = GalleryConfig.from_yaml(config)
elif not isinstance(config, GalleryConfig):
raise TypeError(
f"config must be GalleryConfig, str, or Path, "
f"got {type(config)}"
)
raise TypeError(f"config must be GalleryConfig, str, or Path, got {type(config)}")
else:
if web_folder is None or sources is None:
raise ValueError(
"Either config or both web_folder and sources "
"must be provided"
)
config = GalleryConfig(
web_folder=web_folder,
sources=sources or []
)
raise ValueError("Either config or both web_folder and sources must be provided")
config = GalleryConfig(web_folder=web_folder, sources=sources or [])
# Validate configuration
if not config.sources:
@@ -104,10 +95,7 @@ def generate(
web_folder_path = Path(config.web_folder)
if not _is_writable(web_folder_path):
if verbose:
print(
f"Error: Cannot write to web_folder: "
f"{config.web_folder}"
)
print(f"Error: Cannot write to web_folder: {config.web_folder}")
return False
# Create gallery root directory
@@ -127,17 +115,12 @@ def generate(
source_subdir = gallery_root / source_to_update.name
if source_subdir.exists():
if verbose:
print(
f"Updating source directory {source_to_update.name}..."
)
print(f"Updating source directory {source_to_update.name}...")
try:
shutil.rmtree(source_subdir)
except Exception as e:
if verbose:
print(
f"Warning: Could not clean source subdirectory "
f"{source_subdir}: {e}"
)
print(f"Warning: Could not clean source subdirectory {source_subdir}: {e}")
return False
try:
@@ -162,8 +145,10 @@ def generate(
return False
# Process sources
# config.sources is always List[GallerySource] after GalleryConfig.__post_init__ normalizes it.
source_subdirs = []
for source in config.sources:
source = cast(GallerySource, source)
# Skip sources not matching the update target (if specified)
if source_to_update and source.name != source_to_update.name:
# Still include them in the index if they exist
@@ -178,10 +163,7 @@ def generate(
# Validate source exists
if not source_path.exists():
if verbose:
print(
f"Warning: Source {source.path} does not exist. "
f"Skipping."
)
print(f"Warning: Source {source.path} does not exist. Skipping.")
continue
source_web_dir = gallery_root / source.name
@@ -189,47 +171,37 @@ def generate(
source_web_dir.mkdir(parents=True, exist_ok=True)
except Exception as e:
if verbose:
print(
f"Warning: Could not create directory "
f"{source_web_dir}: {e}"
)
print(f"Warning: Could not create directory {source_web_dir}: {e}")
continue
source_subdirs.append(source.name)
# Process source
if source_path.is_file() and source_path.suffix == '.pdf':
if source_path.is_file() and source_path.suffix == ".pdf":
# Single PDF file
from gallery.utils.processing import process_plot_files
item = process_plot_files(
config=config,
plot_file=source_path,
web_dir=source_web_dir,
)
from gallery.utils.processing import render_gallery_page
render_gallery_page(
config=config,
template=template,
web_dir=source_web_dir,
items=[item],
subdirs=[],
relative_path=Path(source.name)
relative_path=Path(source.name),
)
elif source_path.is_dir():
# Directory of plots
build_gallery(
config,
source_path,
source_web_dir,
template,
Path(source.name)
)
build_gallery(config, source_path, source_web_dir, template, Path(source.name))
else:
if verbose:
print(
f"Warning: Source {source.path} is neither a "
f"directory nor a PDF file. Skipping."
)
print(f"Warning: Source {source.path} is neither a directory nor a PDF file. Skipping.")
continue
if verbose:
@@ -237,15 +209,13 @@ def generate(
except Exception as e:
if verbose:
print(
f"Warning: Error processing source "
f"{source.name}: {e}"
)
print(f"Warning: Error processing source {source.name}: {e}")
continue
# Render gallery root index
try:
from gallery.utils.processing import render_gallery_page
render_gallery_page(
config=config,
template=template,
@@ -253,7 +223,7 @@ def generate(
items=[],
subdirs=source_subdirs,
relative_path=Path("."),
title="Gallery Root"
title="Gallery Root",
)
except Exception as e:
if verbose:
@@ -277,8 +247,8 @@ def generate(
def _count_gallery_plots(gallery_root: Path) -> int:
"""Recursively count plot files (PDFs and HTMLs, excluding index.html) in the gallery output."""
count = 0
for f in gallery_root.rglob('*'):
if f.is_file() and f.suffix.lower() in ('.pdf', '.html') and f.name != 'index.html':
for f in gallery_root.rglob("*"):
if f.is_file() and f.suffix.lower() in (".pdf", ".html") and f.name != "index.html":
count += 1
return count
gallery/builder.py
+13 -17
View File
@@ -2,7 +2,8 @@
import shutil
from pathlib import Path
from typing import Dict, Any, Optional, Union
from typing import Any, Dict, Optional, Union
from jinja2 import Environment, FileSystemLoader, Template
from gallery.config import GalleryConfig
@@ -17,7 +18,6 @@ from gallery.utils.metadata import (
)
from gallery.utils.processing import (
process_plot_files,
needs_update,
render_gallery_page,
)
@@ -39,13 +39,14 @@ def get_template(template_dir: Optional[Union[Path, str]] = None):
if template_dir is None:
# Use package-included template
import gallery
gallery_module_path = Path(gallery.__file__).parent
template_dir = gallery_module_path / "templates"
env = Environment(loader=FileSystemLoader(str(template_dir)))
env.filters['datetime_from_timestamp'] = datetime_from_timestamp
env.filters['strftime'] = strftime_filter
env.filters["datetime_from_timestamp"] = datetime_from_timestamp
env.filters["strftime"] = strftime_filter
return env.get_template("gallery.html")
@@ -54,8 +55,8 @@ def build_gallery(
config: GalleryConfig,
source_dir: Path,
web_dir: Path,
template: Template = None,
relative_path: Path = None,
template: Optional[Template] = None,
relative_path: Optional[Path] = None,
inherited_metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""
@@ -119,7 +120,7 @@ def build_gallery(
subdir_web,
template,
subdir_relative,
current_metadata if config.inherit_from_parent else {}
current_metadata if config.inherit_from_parent else {},
)
subdir_names.append(subdir.name)
@@ -130,15 +131,11 @@ def build_gallery(
items=items,
subdirs=subdir_names,
relative_path=relative_path,
metadata=current_metadata
metadata=current_metadata,
)
def copy_assets(
config: GalleryConfig,
assets_src: Optional[Path] = None,
verbose: bool = False
) -> bool:
def copy_assets(config: GalleryConfig, assets_src: Optional[Path] = None, verbose: bool = False) -> bool:
"""
Copy assets to the web directory.
@@ -154,14 +151,13 @@ def copy_assets(
if assets_src is None:
# Use package-included assets
import gallery
gallery_module_path = Path(gallery.__file__).parent
assets_src = gallery_module_path / "assets"
if not assets_src.exists():
if verbose:
print(
f"Warning: Assets directory {assets_src} not found"
)
print(f"Warning: Assets directory {assets_src} not found")
return False
gallery_root = Path(config.web_folder) / config.plot_root
@@ -171,7 +167,7 @@ def copy_assets(
# so any change to any JS/CSS file triggers a redeploy.
sentinel_dst = assets_dst / "css" / "main.css"
newest_src_mtime = max(
(f.stat().st_mtime for f in assets_src.rglob('*') if f.is_file()),
(f.stat().st_mtime for f in assets_src.rglob("*") if f.is_file()),
default=0,
)
dst_mtime = sentinel_dst.stat().st_mtime if sentinel_dst.exists() else 0
gallery/cli.py
+42 -23
View File
@@ -10,13 +10,17 @@ import argparse
import os
import sys
from pathlib import Path
from typing import List, Optional, cast
import argcomplete
from gallery import generate
from gallery.config import (
ConfigManager, GalleryConfig, GallerySource,
default_config_path, get_active_config_path, ensure_user_config,
ConfigManager,
GalleryConfig,
GallerySource,
ensure_user_config,
get_active_config_path,
)
_WELCOME = """\
@@ -58,6 +62,11 @@ Config file: {config_path}
"""
def _set_completer(action: argparse.Action, completer) -> None:
"""argcomplete reads `.completer` dynamically; argparse.Action has no such attribute."""
setattr(action, "completer", completer)
def _is_configured(config_path: Path) -> bool:
"""Return True if web_folder is set to a non-empty value."""
try:
@@ -104,13 +113,16 @@ def build_parser() -> argparse.ArgumentParser:
description="Scientific Plot Gallery Generator",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--config",
type=str,
default=None,
metavar="FILE",
help="Path to config file (default: package bundled config)",
).completer = argcomplete.completers.FilesCompleter(["yaml", "yml"])
_set_completer(
parser.add_argument(
"--config",
type=str,
default=None,
metavar="FILE",
help="Path to config file (default: package bundled config)",
),
argcomplete.completers.FilesCompleter(["yaml", "yml"]),
)
sub = parser.add_subparsers(dest="command", metavar="COMMAND")
@@ -121,13 +133,16 @@ def build_parser() -> argparse.ArgumentParser:
description="Generate scientific gallery from plot collections",
)
gen.add_argument("--clean", action="store_true", help="Clean gallery directory before generation")
gen.add_argument(
"--source",
type=str,
default=None,
metavar="DIR",
help="Only recompute a specific source directory (name defaults to dir name)",
).completer = argcomplete.completers.DirectoriesCompleter()
_set_completer(
gen.add_argument(
"--source",
type=str,
default=None,
metavar="DIR",
help="Only recompute a specific source directory (name defaults to dir name)",
),
argcomplete.completers.DirectoriesCompleter(),
)
gen.add_argument("-v", "--verbose", action="store_true", help="Print verbose output")
# --- config -------------------------------------------------------------
@@ -146,20 +161,21 @@ def build_parser() -> argparse.ArgumentParser:
cfg_sub.add_parser("sources", help="List all configured sources")
p_get = cfg_sub.add_parser("get", help="Get a config value")
p_get.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi").completer = _config_keys
_set_completer(p_get.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi"), _config_keys)
p_set = cfg_sub.add_parser("set", help="Set a config value")
p_set.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi").completer = _config_keys
_set_completer(p_set.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi"), _config_keys)
p_set.add_argument("value", help="New value (YAML-parsed: use true/false for bools)")
p_add = cfg_sub.add_parser("add-source", help="Add a plot source")
p_add.add_argument("--name", default=None, help="Source name (default: bottom-level directory name)")
p_add.add_argument("--path", required=True, metavar="DIR", help="Path to source directory").completer = (
argcomplete.completers.DirectoriesCompleter()
_set_completer(
p_add.add_argument("--path", required=True, metavar="DIR", help="Path to source directory"),
argcomplete.completers.DirectoriesCompleter(),
)
p_rm = cfg_sub.add_parser("remove-source", help="Remove a plot source by name")
p_rm.add_argument("name", help="Source name to remove").completer = _source_names
_set_completer(p_rm.add_argument("name", help="Source name to remove"), _source_names)
return parser
@@ -174,10 +190,12 @@ def _run_generate(args: argparse.Namespace) -> int:
try:
config = GalleryConfig.from_yaml(config_path)
source_to_update = None
source_to_update: Optional[GallerySource] = None
if args.source:
source_path = Path(args.source).resolve()
matching = next((s for s in config.sources if Path(s.path).resolve() == source_path), None)
# config.sources is always List[GallerySource] after GalleryConfig.__post_init__ normalizes it.
typed_sources = cast(List[GallerySource], config.sources)
matching = next((s for s in typed_sources if Path(s.path).resolve() == source_path), None)
if matching is None:
source_to_update = GallerySource(name=source_path.name, path=source_path)
config.sources.append(source_to_update)
@@ -312,6 +330,7 @@ def main():
sys.exit(_run_install_completion())
elif args.command == "tui":
from gallery.tui import GalleryTUI
config_path = Path(args.config) if args.config else None
GalleryTUI(config_path=config_path).run()
sys.exit(0)
gallery/config/__init__.py
+19 -9
View File
@@ -8,7 +8,8 @@ defaults for gallery generation settings.
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Optional, Union, cast
import yaml
from platformdirs import user_config_dir
@@ -31,8 +32,8 @@ def user_config_path() -> Path:
def get_active_config_path() -> Path:
"""Return the config path to use, with this precedence:
1. User config (~/.config/gallery/config.yaml) — if it exists
2. Package-bundled template — fallback
1. User config (~/.config/gallery/config.yaml) — if it exists
2. Package-bundled template — fallback
"""
ucp = user_config_path()
return ucp if ucp.exists() else default_config_path()
@@ -133,6 +134,7 @@ class ConfigManager:
@dataclass
class GalleryDefaults:
"""Default values for gallery generation."""
png_dpi: int = 400
plot_root: str = "gallery"
cache_enabled: bool = True
@@ -142,6 +144,7 @@ class GalleryDefaults:
@dataclass
class GallerySource:
"""Represents a single data source for the gallery."""
name: str
path: Union[str, Path]
@@ -157,6 +160,7 @@ class GalleryConfig:
Can be created programmatically or loaded from YAML.
"""
web_folder: Union[str, Path]
sources: List[Union[GallerySource, Dict[str, Any]]] = field(default_factory=list)
png_dpi: int = GalleryDefaults.png_dpi
@@ -169,10 +173,11 @@ class GalleryConfig:
if isinstance(self.web_folder, str):
self.web_folder = Path(self.web_folder)
normalized_sources = []
normalized_sources: List[Union[GallerySource, Dict[str, Any]]] = []
for source in self.sources:
if isinstance(source, dict):
source = GallerySource(**source)
source_dict = cast(Dict[str, Any], source)
source = GallerySource(name=source_dict["name"], path=source_dict["path"])
elif not isinstance(source, GallerySource):
raise TypeError(f"Source must be dict or GallerySource, got {type(source)}")
normalized_sources.append(source)
@@ -197,15 +202,19 @@ class GalleryConfig:
gallery_cfg = data.get("gallery", {})
sources_data = data.get("sources", [])
sources = [{"name": s["name"], "path": s["path"]} for s in sources_data]
sources: List[Union[GallerySource, Dict[str, Any]]] = [
{"name": s["name"], "path": s["path"]} for s in sources_data
]
metadata_cfg = data.get("metadata", {})
return cls(
web_folder=web_folder,
sources=sources,
png_dpi=gallery_cfg.get("png_dpi", GalleryDefaults.png_dpi),
plot_root=gallery_cfg.get("plot_root", GalleryDefaults.plot_root),
cache_enabled=data.get("metadata", {}).get("cache_enabled", GalleryDefaults.cache_enabled),
inherit_from_parent=data.get("metadata", {}).get("inherit_from_parent", GalleryDefaults.inherit_from_parent),
cache_enabled=metadata_cfg.get("cache_enabled", GalleryDefaults.cache_enabled),
inherit_from_parent=metadata_cfg.get("inherit_from_parent", GalleryDefaults.inherit_from_parent),
backup_folder=gallery_cfg.get("backup_folder", ""),
)
@@ -231,7 +240,8 @@ class GalleryConfig:
"inherit_from_parent": self.inherit_from_parent,
"supported_formats": [".yaml", ".yml", ".json"],
},
"sources": [{"name": s.name, "path": str(s.path)} for s in self.sources],
# self.sources is always List[GallerySource] after __post_init__ normalizes it.
"sources": [{"name": s.name, "path": str(s.path)} for s in cast(List[GallerySource], self.sources)],
}
with open(yaml_path, "w") as f:
gallery/tui.py
+17 -16
View File
@@ -27,6 +27,7 @@ from textual.app import App, ComposeResult
from textual.binding import Binding
from textual.containers import Horizontal, ScrollableContainer, Vertical
from textual.reactive import reactive
from textual.widget import Widget
from textual.widgets import Button, Collapsible, Footer, Header, Input, Label, RichLog, Static
from gallery.config import ConfigManager, ensure_user_config, get_active_config_path
@@ -37,12 +38,12 @@ from gallery.config import ConfigManager, ensure_user_config, get_active_config_
# widget_id is used as the HTML-style id (#web-folder) in CSS selectors
# ---------------------------------------------------------------------------
CONFIG_FIELDS = [
("web-folder", "paths.web_folder", True),
("plot-root", "gallery.plot_root", False),
("png-dpi", "gallery.png_dpi", False),
("backup-folder", "gallery.backup_folder", False),
("cache-enabled", "metadata.cache_enabled", False),
("inherit-meta", "metadata.inherit_from_parent",False),
("web-folder", "paths.web_folder", True),
("plot-root", "gallery.plot_root", False),
("png-dpi", "gallery.png_dpi", False),
("backup-folder", "gallery.backup_folder", False),
("cache-enabled", "metadata.cache_enabled", False),
("inherit-meta", "metadata.inherit_from_parent", False),
]
REQUIRED_IDS = {fid for fid, _, req in CONFIG_FIELDS if req}
@@ -269,9 +270,9 @@ class GalleryTUI(App):
# Bottom bar — always visible outside the scroll area
with Horizontal(id="footer-bar"):
yield Static("", id="dirty-indicator")
yield Button("Save Config", id="save-btn", variant="success")
yield Button("Generate", id="generate-btn", variant="primary")
yield Button("Quit", id="quit-btn", variant="error")
yield Button("Save Config", id="save-btn", variant="success")
yield Button("Generate", id="generate-btn", variant="primary")
yield Button("Quit", id="quit-btn", variant="error")
yield Footer()
@@ -280,9 +281,7 @@ class GalleryTUI(App):
# ------------------------------------------------------------------
def on_mount(self) -> None:
self._load_config_into_fields()
self.query_one("#config-path-label", Static).update(
f"Config: {self.config_path}"
)
self.query_one("#config-path-label", Static).update(f"Config: {self.config_path}")
def _make_source_row(self, name: str = "", path: str = "") -> Horizontal:
"""Return a single editable source row widget."""
@@ -368,7 +367,7 @@ class GalleryTUI(App):
self._save_config()
@on(Button.Pressed, "#quit-btn")
def action_quit(self) -> None:
async def action_quit(self) -> None:
self.exit()
@on(Button.Pressed, "#save-btn")
@@ -421,7 +420,9 @@ class GalleryTUI(App):
@on(Button.Pressed, ".source-remove-btn")
def _remove_source_row(self, event: Button.Pressed) -> None:
"""Remove the row whose − button was pressed."""
event.button.parent.remove()
parent = event.button.parent
assert isinstance(parent, Widget), "source-remove button must be mounted inside a source row widget"
parent.remove()
self.dirty = True
# -- Generate --------------------------------------------------------
@@ -461,8 +462,7 @@ class GalleryTUI(App):
status.label = label
# --config is a top-level flag (before the subcommand) in the CLI parser
cmd = [sys.executable, "-m", "gallery.cli",
"--config", str(self.config_path), "generate", "--verbose"]
cmd = [sys.executable, "-m", "gallery.cli", "--config", str(self.config_path), "generate", "--verbose"]
try:
proc = subprocess.Popen(
cmd,
@@ -470,6 +470,7 @@ class GalleryTUI(App):
stderr=subprocess.STDOUT,
text=True,
)
assert proc.stdout is not None, "Popen was called with stdout=PIPE"
for line in proc.stdout:
line = line.rstrip()
if line:
gallery/utils/backup.py
+2 -5
View File
@@ -1,14 +1,11 @@
"""Backup utilities for gallery."""
import zipfile
import datetime
import zipfile
from pathlib import Path
def create_backup(
web_folder: Path,
backup_folder: Path
) -> bool:
def create_backup(web_folder: Path, backup_folder: Path) -> bool:
"""
Create a backup of the web folder.
gallery/utils/metadata.py
+15 -24
View File
@@ -13,9 +13,10 @@ Features:
"""
import json
import yaml
from pathlib import Path
from typing import Dict, Any
from typing import Any, Dict
import yaml
def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
@@ -33,15 +34,14 @@ def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
return {}
try:
with metadata_path.open('r', encoding='utf-8') as f:
with metadata_path.open("r", encoding="utf-8") as f:
suffix_lower = metadata_path.suffix.lower()
if suffix_lower == '.yaml' or suffix_lower == '.yml':
if suffix_lower == ".yaml" or suffix_lower == ".yml":
return yaml.safe_load(f) or {}
elif metadata_path.suffix.lower() == '.json':
elif metadata_path.suffix.lower() == ".json":
return json.load(f) or {}
else:
print(f"Warning: Unknown metadata file format: "
f"{metadata_path}")
print(f"Warning: Unknown metadata file format: {metadata_path}")
return {}
except (yaml.YAMLError, json.JSONDecodeError, IOError) as e:
print(f"Warning: Could not parse metadata file {metadata_path}: {e}")
@@ -59,7 +59,7 @@ def load_folder_metadata(folder_path: Path) -> Dict[str, Any]:
Dictionary containing the folder metadata
"""
# Try YAML first, then JSON for backwards compatibility
for filename in ['metadata.yaml', 'metadata.yml', 'metadata.json']:
for filename in ["metadata.yaml", "metadata.yml", "metadata.json"]:
metadata_path = folder_path / filename
if metadata_path.exists():
@@ -81,7 +81,7 @@ def get_metadata_file_path(folder_path: Path) -> str:
String path to the metadata file (existing or suggested)
"""
# Preferred order: YAML first, then JSON
preferred_files = ['metadata.yaml', 'metadata.yml', 'metadata.json']
preferred_files = ["metadata.yaml", "metadata.yml", "metadata.json"]
for filename in preferred_files:
metadata_path = folder_path / filename
@@ -89,13 +89,10 @@ def get_metadata_file_path(folder_path: Path) -> str:
return str(metadata_path)
# If no file exists, suggest metadata.yaml (preferred format)
return str(folder_path / 'metadata.yaml')
return str(folder_path / "metadata.yaml")
def merge_metadata(
parent_metadata: Dict[str, Any],
child_metadata: Dict[str, Any]
) -> Dict[str, Any]:
def merge_metadata(parent_metadata: Dict[str, Any], child_metadata: Dict[str, Any]) -> Dict[str, Any]:
"""
Merge parent and child metadata, with child values overriding parent.
@@ -111,10 +108,7 @@ def merge_metadata(
return merged
def resolve_metadata_for_plot(
plot_path: Path,
inherited_metadata: Dict[str, Any]
) -> Dict[str, Any]:
def resolve_metadata_for_plot(plot_path: Path, inherited_metadata: Dict[str, Any]) -> Dict[str, Any]:
"""
Resolve metadata for a specific plot.
@@ -131,7 +125,7 @@ def resolve_metadata_for_plot(
plot_dir = plot_path.parent
# Check for plot-specific metadata files
for suffix in ['.yaml', '.yml', '.json']:
for suffix in [".yaml", ".yml", ".json"]:
plot_metadata_path = plot_dir / f"{plot_stem}{suffix}"
if plot_metadata_path.exists():
plot_metadata = load_metadata_file(plot_metadata_path)
@@ -141,10 +135,7 @@ def resolve_metadata_for_plot(
return inherited_metadata.copy()
def save_metadata_cache(
web_dir: Path,
plot_metadata_cache: Dict[str, Dict[str, Any]]
) -> None:
def save_metadata_cache(web_dir: Path, plot_metadata_cache: Dict[str, Dict[str, Any]]) -> None:
"""
Save plot metadata cache to meta_cache.json in the web directory.
@@ -154,7 +145,7 @@ def save_metadata_cache(
"""
cache_path = web_dir / "meta_cache.json"
try:
with cache_path.open('w', encoding='utf-8') as f:
with cache_path.open("w", encoding="utf-8") as f:
json.dump(plot_metadata_cache, f, indent=2, ensure_ascii=False)
except IOError as e:
print(f"Warning: Could not save metadata cache {cache_path}: {e}")
gallery/utils/processing.py
+39 -38
View File
@@ -3,34 +3,31 @@
import shutil
import subprocess
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, Optional
from jinja2 import Template
from gallery.config import GalleryConfig
from gallery.utils.metadata import (
get_metadata_file_path,
resolve_metadata_for_plot,
)
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
)
try:
import fitz # PyMuPDF
_PYMUPDF_AVAILABLE = True
except ImportError:
_PYMUPDF_AVAILABLE = False
_IMAGEMAGICK_AVAILABLE = shutil.which("convert") is not None
from jinja2 import Template
from gallery.utils.metadata import (
resolve_metadata_for_plot,
get_metadata_file_path,
)
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
)
from gallery.config import GalleryConfig
def process_html_file(
html_file: Path,
web_dir: Path,
current_metadata: Dict[str, Any] = None
) -> dict:
def process_html_file(html_file: Path, web_dir: Path, current_metadata: Optional[Dict[str, Any]] = None) -> dict:
"""
Process HTML plot file, copying it to web directory.
@@ -60,15 +57,15 @@ def process_html_file(
"html_href": html_file.name,
"is_html": True,
"metadata": plot_metadata,
"creation_time": source_creation_time
"creation_time": source_creation_time,
}
def process_plot_files(
config: GalleryConfig,
plot_file: Path,
web_dir: Path,
current_metadata: Dict[str, Any] = None,
config: GalleryConfig,
plot_file: Path,
web_dir: Path,
current_metadata: Optional[Dict[str, Any]] = None,
) -> dict:
"""
Process plot files (PDF/PNG or HTML), handling conversion and copying.
@@ -82,7 +79,7 @@ def process_plot_files(
Returns:
Dictionary containing plot information
"""
if plot_file.suffix.lower() == '.html':
if plot_file.suffix.lower() == ".html":
return process_html_file(plot_file, web_dir, current_metadata)
# Handle PDF files
@@ -111,7 +108,7 @@ def process_plot_files(
"png_href": png_file.name,
"is_html": False,
"metadata": plot_metadata,
"creation_time": source_creation_time
"creation_time": source_creation_time,
}
@@ -122,8 +119,8 @@ def render_gallery_page(
items: list,
subdirs: list,
relative_path: Path,
title: str = None,
metadata: dict = None
title: Optional[str] = None,
metadata: Optional[dict] = None,
) -> None:
"""
Unified template rendering for all gallery pages.
@@ -139,8 +136,7 @@ def render_gallery_page(
metadata: Metadata dictionary (optional)
"""
if title is None:
title = "Gallery" if relative_path == Path(
".") else f"Gallery: {relative_path}"
title = "Gallery" if relative_path == Path(".") else f"Gallery: {relative_path}"
if metadata is None:
metadata = {}
@@ -151,7 +147,7 @@ def render_gallery_page(
"file_count": len(items),
"folder_count": len(subdirs),
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
"total_size_bytes": current_stats["total_size"],
}
# Calculate relative path to assets
@@ -185,7 +181,7 @@ def render_gallery_page(
folder_metadata=metadata,
assets_path=assets_path,
source_dir=str(web_dir),
metadata_file_path=get_metadata_file_path(web_dir)
metadata_file_path=get_metadata_file_path(web_dir),
)
f.write(rendered_html)
@@ -232,13 +228,18 @@ def _convert_pdf_pymupdf(pdf_path: Path, png_path: Path, dpi: int) -> None:
def _convert_pdf_imagemagick(pdf_path: Path, png_path: Path, dpi: int) -> None:
subprocess.run([
"convert",
"-density", str(dpi),
str(pdf_path),
"-quality", "95",
str(png_path),
], check=True)
subprocess.run(
[
"convert",
"-density",
str(dpi),
str(pdf_path),
"-quality",
"95",
str(png_path),
],
check=True,
)
def needs_update(source_file: Path, target_file: Path) -> bool:
gallery/utils/stats.py
+2 -2
View File
@@ -30,9 +30,9 @@ def calculate_directory_stats(directory: Path) -> dict:
size = item.stat().st_size
stats["total_size"] += size
if item.suffix.lower() == '.pdf':
if item.suffix.lower() == ".pdf":
stats["pdf_size"] += size
elif item.suffix.lower() == '.png':
elif item.suffix.lower() == ".png":
stats["png_size"] += size
elif item.is_dir():
stats["folder_count"] += 1