lars/ETPlot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite CI to lint/typecheck/audit/test only; add HPC deployment path

Browse Source 
- Replace the Docker build/publish CI stages with ruff (lint + format
  check), ty (type check), pip-audit, and pytest run directly against
  python:3.11-slim; Docker remains for manual/server deployment only.
- Swap black/pylint/mypy for ruff/ty across pyproject.toml, and fix
  every resulting lint, format, and type diagnostic in gallery/ and
  plotstyle/.
- Fix tests broken/stale from before the package restructuring: wrong
  `utils.*` import paths, mock patch targets pointed at the wrong
  module, and PDF-conversion tests still assuming ImageMagick instead
  of the current PyMuPDF-first path. Drop test_container.py (obsolete
  Docker-container smoke tests, fully superseded elsewhere).
- Add a plain-venv + systemd --user timer deployment path
  (deploy/systemd/) for HPC login nodes without a Docker daemon, where
  public_html is already served by existing infrastructure.
- Document both in CLAUDE.md, including running CI's checks locally
  before committing.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Lars Bogner
2026-07-22 15:29:55 +02:00
parent 4ba321b327
commit 3b9d1ef1a8
27 changed files with 752 additions and 915 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gallery/utils/backup.py
+2 -5
View File
@@ -1,14 +1,11 @@
"""Backup utilities for gallery."""
import zipfile
import datetime
import zipfile
from pathlib import Path
def create_backup(
web_folder: Path,
backup_folder: Path
) -> bool:
def create_backup(web_folder: Path, backup_folder: Path) -> bool:
"""
Create a backup of the web folder.
gallery/utils/metadata.py
+15 -24
View File
@@ -13,9 +13,10 @@ Features:
"""
import json
import yaml
from pathlib import Path
from typing import Dict, Any
from typing import Any, Dict
import yaml
def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
@@ -33,15 +34,14 @@ def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
return {}
try:
with metadata_path.open('r', encoding='utf-8') as f:
with metadata_path.open("r", encoding="utf-8") as f:
suffix_lower = metadata_path.suffix.lower()
if suffix_lower == '.yaml' or suffix_lower == '.yml':
if suffix_lower == ".yaml" or suffix_lower == ".yml":
return yaml.safe_load(f) or {}
elif metadata_path.suffix.lower() == '.json':
elif metadata_path.suffix.lower() == ".json":
return json.load(f) or {}
else:
print(f"Warning: Unknown metadata file format: "
f"{metadata_path}")
print(f"Warning: Unknown metadata file format: {metadata_path}")
return {}
except (yaml.YAMLError, json.JSONDecodeError, IOError) as e:
print(f"Warning: Could not parse metadata file {metadata_path}: {e}")
@@ -59,7 +59,7 @@ def load_folder_metadata(folder_path: Path) -> Dict[str, Any]:
Dictionary containing the folder metadata
"""
# Try YAML first, then JSON for backwards compatibility
for filename in ['metadata.yaml', 'metadata.yml', 'metadata.json']:
for filename in ["metadata.yaml", "metadata.yml", "metadata.json"]:
metadata_path = folder_path / filename
if metadata_path.exists():
@@ -81,7 +81,7 @@ def get_metadata_file_path(folder_path: Path) -> str:
String path to the metadata file (existing or suggested)
"""
# Preferred order: YAML first, then JSON
preferred_files = ['metadata.yaml', 'metadata.yml', 'metadata.json']
preferred_files = ["metadata.yaml", "metadata.yml", "metadata.json"]
for filename in preferred_files:
metadata_path = folder_path / filename
@@ -89,13 +89,10 @@ def get_metadata_file_path(folder_path: Path) -> str:
return str(metadata_path)
# If no file exists, suggest metadata.yaml (preferred format)
return str(folder_path / 'metadata.yaml')
return str(folder_path / "metadata.yaml")
def merge_metadata(
parent_metadata: Dict[str, Any],
child_metadata: Dict[str, Any]
) -> Dict[str, Any]:
def merge_metadata(parent_metadata: Dict[str, Any], child_metadata: Dict[str, Any]) -> Dict[str, Any]:
"""
Merge parent and child metadata, with child values overriding parent.
@@ -111,10 +108,7 @@ def merge_metadata(
return merged
def resolve_metadata_for_plot(
plot_path: Path,
inherited_metadata: Dict[str, Any]
) -> Dict[str, Any]:
def resolve_metadata_for_plot(plot_path: Path, inherited_metadata: Dict[str, Any]) -> Dict[str, Any]:
"""
Resolve metadata for a specific plot.
@@ -131,7 +125,7 @@ def resolve_metadata_for_plot(
plot_dir = plot_path.parent
# Check for plot-specific metadata files
for suffix in ['.yaml', '.yml', '.json']:
for suffix in [".yaml", ".yml", ".json"]:
plot_metadata_path = plot_dir / f"{plot_stem}{suffix}"
if plot_metadata_path.exists():
plot_metadata = load_metadata_file(plot_metadata_path)
@@ -141,10 +135,7 @@ def resolve_metadata_for_plot(
return inherited_metadata.copy()
def save_metadata_cache(
web_dir: Path,
plot_metadata_cache: Dict[str, Dict[str, Any]]
) -> None:
def save_metadata_cache(web_dir: Path, plot_metadata_cache: Dict[str, Dict[str, Any]]) -> None:
"""
Save plot metadata cache to meta_cache.json in the web directory.
@@ -154,7 +145,7 @@ def save_metadata_cache(
"""
cache_path = web_dir / "meta_cache.json"
try:
with cache_path.open('w', encoding='utf-8') as f:
with cache_path.open("w", encoding="utf-8") as f:
json.dump(plot_metadata_cache, f, indent=2, ensure_ascii=False)
except IOError as e:
print(f"Warning: Could not save metadata cache {cache_path}: {e}")
gallery/utils/processing.py
+39 -38
View File
@@ -3,34 +3,31 @@
import shutil
import subprocess
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, Optional
from jinja2 import Template
from gallery.config import GalleryConfig
from gallery.utils.metadata import (
get_metadata_file_path,
resolve_metadata_for_plot,
)
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
)
try:
import fitz # PyMuPDF
_PYMUPDF_AVAILABLE = True
except ImportError:
_PYMUPDF_AVAILABLE = False
_IMAGEMAGICK_AVAILABLE = shutil.which("convert") is not None
from jinja2 import Template
from gallery.utils.metadata import (
resolve_metadata_for_plot,
get_metadata_file_path,
)
from gallery.utils.stats import (
calculate_directory_stats,
format_file_size,
)
from gallery.config import GalleryConfig
def process_html_file(
html_file: Path,
web_dir: Path,
current_metadata: Dict[str, Any] = None
) -> dict:
def process_html_file(html_file: Path, web_dir: Path, current_metadata: Optional[Dict[str, Any]] = None) -> dict:
"""
Process HTML plot file, copying it to web directory.
@@ -60,15 +57,15 @@ def process_html_file(
"html_href": html_file.name,
"is_html": True,
"metadata": plot_metadata,
"creation_time": source_creation_time
"creation_time": source_creation_time,
}
def process_plot_files(
config: GalleryConfig,
plot_file: Path,
web_dir: Path,
current_metadata: Dict[str, Any] = None,
config: GalleryConfig,
plot_file: Path,
web_dir: Path,
current_metadata: Optional[Dict[str, Any]] = None,
) -> dict:
"""
Process plot files (PDF/PNG or HTML), handling conversion and copying.
@@ -82,7 +79,7 @@ def process_plot_files(
Returns:
Dictionary containing plot information
"""
if plot_file.suffix.lower() == '.html':
if plot_file.suffix.lower() == ".html":
return process_html_file(plot_file, web_dir, current_metadata)
# Handle PDF files
@@ -111,7 +108,7 @@ def process_plot_files(
"png_href": png_file.name,
"is_html": False,
"metadata": plot_metadata,
"creation_time": source_creation_time
"creation_time": source_creation_time,
}
@@ -122,8 +119,8 @@ def render_gallery_page(
items: list,
subdirs: list,
relative_path: Path,
title: str = None,
metadata: dict = None
title: Optional[str] = None,
metadata: Optional[dict] = None,
) -> None:
"""
Unified template rendering for all gallery pages.
@@ -139,8 +136,7 @@ def render_gallery_page(
metadata: Metadata dictionary (optional)
"""
if title is None:
title = "Gallery" if relative_path == Path(
".") else f"Gallery: {relative_path}"
title = "Gallery" if relative_path == Path(".") else f"Gallery: {relative_path}"
if metadata is None:
metadata = {}
@@ -151,7 +147,7 @@ def render_gallery_page(
"file_count": len(items),
"folder_count": len(subdirs),
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
"total_size_bytes": current_stats["total_size"],
}
# Calculate relative path to assets
@@ -185,7 +181,7 @@ def render_gallery_page(
folder_metadata=metadata,
assets_path=assets_path,
source_dir=str(web_dir),
metadata_file_path=get_metadata_file_path(web_dir)
metadata_file_path=get_metadata_file_path(web_dir),
)
f.write(rendered_html)
@@ -232,13 +228,18 @@ def _convert_pdf_pymupdf(pdf_path: Path, png_path: Path, dpi: int) -> None:
def _convert_pdf_imagemagick(pdf_path: Path, png_path: Path, dpi: int) -> None:
subprocess.run([
"convert",
"-density", str(dpi),
str(pdf_path),
"-quality", "95",
str(png_path),
], check=True)
subprocess.run(
[
"convert",
"-density",
str(dpi),
str(pdf_path),
"-quality",
"95",
str(png_path),
],
check=True,
)
def needs_update(source_file: Path, target_file: Path) -> bool:
gallery/utils/stats.py
+2 -2
View File
@@ -30,9 +30,9 @@ def calculate_directory_stats(directory: Path) -> dict:
size = item.stat().st_size
stats["total_size"] += size
if item.suffix.lower() == '.pdf':
if item.suffix.lower() == ".pdf":
stats["pdf_size"] += size
elif item.suffix.lower() == '.png':
elif item.suffix.lower() == ".png":
stats["png_size"] += size
elif item.is_dir():
stats["folder_count"] += 1