Refactor: reorganize configuration management and add default config file

This commit is contained in:
Kylian Schmidt
2026-05-08 14:05:40 +02:00
parent 6978a1b768
commit 63e4af0f9f
6 changed files with 408 additions and 233 deletions
+2 -5
View File
@@ -68,9 +68,8 @@ generate() [api.py]
| `gallery/utils/metadata.py` | Load/merge/cache YAML+JSON metadata; per-plot metadata resolution |
| `gallery/utils/stats.py` | Directory size/count statistics |
| `gallery/templates/gallery.html` | Single Jinja2 template for all gallery pages |
| `assets/js/` | Vanilla JS modules loaded as ES modules; `GalleryApp` in `gallery-app.js` orchestrates all managers |
| `assets/css/` | Modular CSS; `main.css` imports all others via `@import` |
| `generate_gallery.py` | Legacy CLI wrapper — kept for backward compatibility |
| `gallery/assets/js/` | Vanilla JS modules loaded as ES modules; `GalleryApp` in `gallery-app.js` orchestrates all managers |
| `gallery/assets/css/` | Modular CSS; `main.css` imports all others via `@import` |
| `config.yaml` | Local deployment config (paths are machine-specific) |
### Config File Format
@@ -105,8 +104,6 @@ When `source_to_update` is passed to `generate()`, only that source's subdirecto
The frontend is vanilla ES modules — no build step. `assets/js/main.js` imports `GalleryApp` from `gallery-app.js`, which instantiates all manager classes (`ThemeManager`, `SearchManager`, `NavigationManager`, etc.). Each manager is self-contained. The template embeds gallery data as JSON in the page; JS reads it at runtime.
Assets are served from `/assets/` relative to gallery pages. The Python code calculates the correct `../` depth per page when rendering the template.
### Deployment
The project ships a `Singularity.def` / `web.sif` Apptainer container for HPC environments. CI (`.gitlab-ci.yml`) builds the container and runs pytest inside it. For local development the `.venv` is sufficient.
+183 -61
View File
@@ -1,7 +1,7 @@
"""
Command-line interface for gallery generation.
Provides a CLI entry point for gallery generation when using git clone setup.
Provides a CLI entry point for gallery generation and config management.
"""
import argparse
@@ -9,113 +9,235 @@ import sys
from pathlib import Path
from gallery import generate
from gallery.config import GalleryConfig
from gallery.config import ConfigManager, GalleryConfig, GallerySource, default_config_path
_WELCOME = """\
Gallery - Scientific Plot Gallery Generator
===========================================
def main():
Generates responsive static HTML galleries from collections of PDFs and HTMLs.
Getting started:
1. Set your web output directory:
gallery config set paths.web_folder /path/to/your/public_html
2. Add one or more plot sources:
gallery config add-source --name my_plots --path /path/to/plots
3. Generate your gallery:
gallery generate
Config commands:
gallery config list Show all settings
gallery config get <key> Get a single value (e.g. gallery.png_dpi)
gallery config set <key> <value> Update a setting (e.g. paths.web_folder /my/web)
gallery config add-source --name X --path P Add a plot source
gallery config remove-source <name> Remove a plot source
gallery config sources List configured sources
gallery config path Show config file location
Generate commands:
gallery generate Generate gallery from config
gallery generate --config myconfig.yaml Use a custom config file
gallery generate --clean Clean and regenerate everything
gallery generate --source /path/to/plots Regenerate one source only
gallery generate --verbose Print detailed output
Config file: {config_path}
"""
Main CLI entry point for gallery generation.
Supports:
- Loading config from YAML file (default: config.yaml)
- Clean gallery directory before generation
- Override to process only a specific source directory
"""
def _is_configured(config_path: Path) -> bool:
"""Return True if web_folder is set to a non-empty value."""
try:
mgr = ConfigManager(config_path)
web_folder = mgr.get("paths.web_folder")
return bool(web_folder and str(web_folder).strip())
except Exception:
return False
# ---------------------------------------------------------------------------
# Config subcommand
# ---------------------------------------------------------------------------
def _cmd_config(argv: list) -> int:
parser = argparse.ArgumentParser(
description='Generate scientific gallery from plot collections',
prog="gallery config",
description="Read and write gallery configuration",
)
parser.add_argument(
"--config",
type=str,
default=None,
help="Path to config file (default: package config)",
)
sub = parser.add_subparsers(dest="action", metavar="ACTION")
sub.add_parser("list", help="Print all config values")
sub.add_parser("path", help="Print the resolved config file path")
p_get = sub.add_parser("get", help="Get a config value by key")
p_get.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi")
p_set = sub.add_parser("set", help="Set a config value")
p_set.add_argument("key", help="Dot-separated key, e.g. gallery.png_dpi")
p_set.add_argument("value", help="New value (YAML-parsed: use true/false for bools)")
p_add = sub.add_parser("add-source", help="Add a plot source")
p_add.add_argument("--name", default=None, help="Source name (default: bottom-level directory name)")
p_add.add_argument("--path", required=True, help="Path to source directory")
p_rm = sub.add_parser("remove-source", help="Remove a plot source by name")
p_rm.add_argument("name", help="Source name to remove")
sub.add_parser("sources", help="List all configured sources")
args = parser.parse_args(argv)
config_path = Path(args.config) if args.config else default_config_path()
mgr = ConfigManager(config_path)
if args.action == "path":
print(mgr.path)
elif args.action == "list":
import yaml
print(yaml.dump(mgr.list_all(), default_flow_style=False).rstrip())
elif args.action == "get":
try:
value = mgr.get(args.key)
print(value)
except KeyError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
elif args.action == "set":
try:
mgr.set(args.key, args.value)
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
return 1
elif args.action == "add-source":
try:
name = args.name or Path(args.path).resolve().name
mgr.add_source(name, args.path)
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
elif args.action == "remove-source":
try:
mgr.remove_source(args.name)
except KeyError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
elif args.action == "sources":
sources = mgr.list_sources()
if not sources:
print("No sources configured.")
else:
for s in sources:
print(f" {s['name']}: {s['path']}")
else:
parser.print_help()
return 0
# ---------------------------------------------------------------------------
# Generate subcommand (existing behaviour)
# ---------------------------------------------------------------------------
def _cmd_generate(argv: list) -> int:
parser = argparse.ArgumentParser(
prog="gallery generate",
description="Generate scientific gallery from plot collections",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
gallery # Generate from config.yaml
gallery --config myconfig.yaml # Use custom config file
gallery --clean # Clean and regenerate
gallery --source /path/to/plots # Generate only specific source
"""
gallery generate # Generate using package config
gallery generate --config myconfig.yaml # Use custom config file
gallery generate --clean # Clean and regenerate
gallery generate --source /path/to/plots # Generate only specific source
""",
)
parser.add_argument(
'--config',
type=str,
default='config.yaml',
help='Path to config.yaml file (default: config.yaml)'
)
parser.add_argument(
'--clean',
action='store_true',
help='Clean gallery directory before generation'
)
parser.add_argument(
'--source',
"--config",
type=str,
default=None,
help='Override to only recompute a specific source directory. '
'If the directory is not in config, it will be added '
'temporarily.'
help="Path to config.yaml (default: package bundled config)",
)
parser.add_argument("--clean", action="store_true", help="Clean gallery directory before generation")
parser.add_argument(
'-v', '--verbose',
action='store_true',
help='Print verbose output'
"--source",
type=str,
default=None,
help="Only recompute a specific source directory",
)
parser.add_argument("-v", "--verbose", action="store_true", help="Print verbose output")
args = parser.parse_args()
args = parser.parse_args(argv)
config_path = Path(args.config) if args.config else default_config_path()
try:
# Load config from file
config = GalleryConfig.from_yaml(args.config)
config = GalleryConfig.from_yaml(config_path)
# Handle source override
source_to_update = None
if args.source:
from gallery.config import GallerySource
source_path = Path(args.source).resolve()
# Check if source is in config
matching_source = None
for source in config.sources:
if Path(source.path).resolve() == source_path:
matching_source = source
break
# If not in config, create a temporary source entry
if matching_source is None:
source_name = source_path.name
source_to_update = GallerySource(
name=source_name,
path=source_path
)
source_to_update = GallerySource(name=source_name, path=source_path)
config.sources.append(source_to_update)
if args.verbose:
print(
f"Source {args.source} not in config. "
f"Adding temporarily as '{source_name}'"
)
print(f"Source {args.source} not in config. Adding temporarily as '{source_name}'")
else:
source_to_update = matching_source
# Generate gallery
success = generate(
config=config,
clean_first=args.clean,
verbose=args.verbose,
source_to_update=source_to_update
source_to_update=source_to_update,
)
sys.exit(0 if success else 1)
return 0 if success else 1
except FileNotFoundError as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
return 1
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
return 1
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
return 1
if __name__ == '__main__':
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main():
if len(sys.argv) > 1 and sys.argv[1] == "config":
sys.exit(_cmd_config(sys.argv[2:]))
elif len(sys.argv) > 1 and sys.argv[1] == "generate":
sys.exit(_cmd_generate(sys.argv[2:]))
else:
print(_WELCOME.format(config_path=default_config_path()))
sys.exit(0)
if __name__ == "__main__":
main()
-164
View File
@@ -1,164 +0,0 @@
"""
Configuration Management for Scientific Gallery Generator
This module provides dataclasses for managing configuration, including
defaults for gallery generation settings.
"""
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Dict, Any, Union
import yaml
@dataclass
class GalleryDefaults:
"""Default values for gallery generation."""
png_dpi: int = 400
plot_root: str = "gallery"
cache_enabled: bool = True
inherit_from_parent: bool = True
@dataclass
class GallerySource:
"""Represents a single data source for the gallery."""
name: str
path: Union[str, Path]
def __post_init__(self):
"""Convert path to Path object if needed."""
if isinstance(self.path, str):
self.path = Path(self.path)
@dataclass
class GalleryConfig:
"""
Main configuration for gallery generation.
Can be created programmatically or loaded from YAML.
"""
web_folder: Union[str, Path]
sources: List[Union[GallerySource, Dict[str, Any]]] = field(
default_factory=list
)
png_dpi: int = GalleryDefaults.png_dpi
plot_root: str = GalleryDefaults.plot_root
cache_enabled: bool = GalleryDefaults.cache_enabled
inherit_from_parent: bool = GalleryDefaults.inherit_from_parent
backup_folder: str = ""
def __post_init__(self):
"""Normalize and validate configuration."""
# Convert web_folder to Path
if isinstance(self.web_folder, str):
self.web_folder = Path(self.web_folder)
# Convert sources to GallerySource objects if they're dicts
normalized_sources = []
for source in self.sources:
if isinstance(source, dict):
source = GallerySource(**source)
elif not isinstance(source, GallerySource):
raise TypeError(
f"Source must be dict or GallerySource, "
f"got {type(source)}"
)
normalized_sources.append(source)
self.sources = normalized_sources
@classmethod
def from_yaml(cls, yaml_file: Union[str, Path]) -> "GalleryConfig":
"""
Load configuration from a YAML file.
Args:
yaml_file: Path to the YAML configuration file
Returns:
GalleryConfig instance with loaded settings
Raises:
FileNotFoundError: If the YAML file doesn't exist
yaml.YAMLError: If the YAML file is malformed
"""
yaml_path = Path(yaml_file)
if not yaml_path.exists():
raise FileNotFoundError(f"Config file not found: {yaml_file}")
with open(yaml_path, "r") as f:
data = yaml.safe_load(f)
if data is None:
data = {}
# Extract relevant sections
web_folder = data.get("paths", {}).get("web_folder")
if not web_folder:
raise ValueError(
"web_folder must be specified in config under paths"
)
gallery_cfg = data.get("gallery", {})
sources_data = data.get("sources", [])
# Build sources list
sources = [
{"name": s["name"], "path": s["path"]} for s in sources_data
]
return cls(
web_folder=web_folder,
sources=sources,
png_dpi=gallery_cfg.get("png_dpi", GalleryDefaults.png_dpi),
plot_root=gallery_cfg.get(
"plot_root", GalleryDefaults.plot_root
),
cache_enabled=data.get("metadata", {}).get(
"cache_enabled", GalleryDefaults.cache_enabled
),
inherit_from_parent=data.get("metadata", {}).get(
"inherit_from_parent", GalleryDefaults.inherit_from_parent
),
backup_folder=gallery_cfg.get("backup_folder", ""),
)
def to_yaml(self, yaml_file: Union[str, Path]) -> None:
"""
Save the current configuration to a YAML file.
Args:
yaml_file: Path where to save the YAML configuration
Raises:
IOError: If unable to write to the specified file
"""
yaml_path = Path(yaml_file)
data = {
"paths": {
"work_dir": str(Path.cwd()),
"web_folder": str(self.web_folder),
},
"gallery": {
"plot_root": self.plot_root,
"png_dpi": self.png_dpi,
"backup_folder": self.backup_folder,
},
"ui": {
"max_recent_plots": 20,
"search_debounce_ms": 300,
},
"metadata": {
"cache_enabled": self.cache_enabled,
"inherit_from_parent": self.inherit_from_parent,
"supported_formats": [".yaml", ".yml", ".json"],
},
"sources": [
{"name": s.name, "path": str(s.path)} for s in self.sources
],
}
with open(yaml_path, "w") as f:
yaml.dump(data, f, default_flow_style=False)
+206
View File
@@ -0,0 +1,206 @@
"""
Configuration Management for Scientific Gallery Generator
This module provides dataclasses for managing configuration, including
defaults for gallery generation settings.
"""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
import yaml
def default_config_path() -> Path:
"""Return the path to the package-bundled default config."""
return Path(__file__).parent / "config.yaml"
def _load_raw(path: Path) -> Dict[str, Any]:
with open(path, "r") as f:
return yaml.safe_load(f) or {}
def _save_raw(path: Path, data: Dict[str, Any]) -> None:
with open(path, "w") as f:
yaml.dump(data, f, default_flow_style=False, allow_unicode=True)
def _get_nested(data: Dict, keys: List[str]) -> Any:
for k in keys:
if not isinstance(data, dict) or k not in data:
raise KeyError(f"Key '{'.'.join(keys)}' not found in config")
data = data[k]
return data
def _set_nested(data: Dict, keys: List[str], value: Any) -> None:
for k in keys[:-1]:
data = data.setdefault(k, {})
data[keys[-1]] = value
class ConfigManager:
"""Read/write access to a gallery config YAML file.
Intended to be reused by both the CLI and a future TUI layer.
"""
def __init__(self, config_path: Optional[Path] = None):
self.path = Path(config_path) if config_path else default_config_path()
# ------------------------------------------------------------------
# Core operations
# ------------------------------------------------------------------
def get(self, key: str) -> Any:
"""Return the value at dot-separated *key* (e.g. 'gallery.png_dpi')."""
data = _load_raw(self.path)
return _get_nested(data, key.split("."))
def set(self, key: str, value: str) -> None:
"""Set *key* to *value*, coercing type via YAML parsing."""
data = _load_raw(self.path)
parsed = yaml.safe_load(value)
_set_nested(data, key.split("."), parsed)
_save_raw(self.path, data)
def list_all(self) -> Dict[str, Any]:
"""Return the full config dict."""
return _load_raw(self.path)
# ------------------------------------------------------------------
# Source helpers
# ------------------------------------------------------------------
def add_source(self, name: str, path: Union[str, Path]) -> None:
"""Append a new source entry; raises ValueError if name already exists."""
data = _load_raw(self.path)
sources: List[Dict] = data.setdefault("sources", [])
if any(s.get("name") == name for s in sources):
raise ValueError(f"Source '{name}' already exists")
sources.append({"name": name, "path": str(path)})
_save_raw(self.path, data)
def remove_source(self, name: str) -> None:
"""Remove the source with the given name; raises KeyError if not found."""
data = _load_raw(self.path)
sources: List[Dict] = data.get("sources", [])
filtered = [s for s in sources if s.get("name") != name]
if len(filtered) == len(sources):
raise KeyError(f"Source '{name}' not found")
data["sources"] = filtered
_save_raw(self.path, data)
def list_sources(self) -> List[Dict]:
"""Return the list of source dicts."""
return _load_raw(self.path).get("sources", [])
@dataclass
class GalleryDefaults:
"""Default values for gallery generation."""
png_dpi: int = 400
plot_root: str = "gallery"
cache_enabled: bool = True
inherit_from_parent: bool = True
@dataclass
class GallerySource:
"""Represents a single data source for the gallery."""
name: str
path: Union[str, Path]
def __post_init__(self):
if isinstance(self.path, str):
self.path = Path(self.path)
@dataclass
class GalleryConfig:
"""
Main configuration for gallery generation.
Can be created programmatically or loaded from YAML.
"""
web_folder: Union[str, Path]
sources: List[Union[GallerySource, Dict[str, Any]]] = field(default_factory=list)
png_dpi: int = GalleryDefaults.png_dpi
plot_root: str = GalleryDefaults.plot_root
cache_enabled: bool = GalleryDefaults.cache_enabled
inherit_from_parent: bool = GalleryDefaults.inherit_from_parent
backup_folder: str = ""
def __post_init__(self):
if isinstance(self.web_folder, str):
self.web_folder = Path(self.web_folder)
normalized_sources = []
for source in self.sources:
if isinstance(source, dict):
source = GallerySource(**source)
elif not isinstance(source, GallerySource):
raise TypeError(f"Source must be dict or GallerySource, got {type(source)}")
normalized_sources.append(source)
self.sources = normalized_sources
@classmethod
def from_yaml(cls, yaml_file: Union[str, Path]) -> "GalleryConfig":
"""Load configuration from a YAML file."""
yaml_path = Path(yaml_file)
if not yaml_path.exists():
raise FileNotFoundError(f"Config file not found: {yaml_file}")
with open(yaml_path, "r") as f:
data = yaml.safe_load(f)
if data is None:
data = {}
web_folder = data.get("paths", {}).get("web_folder")
if not web_folder:
raise ValueError("web_folder must be specified in config under paths")
gallery_cfg = data.get("gallery", {})
sources_data = data.get("sources", [])
sources = [{"name": s["name"], "path": s["path"]} for s in sources_data]
return cls(
web_folder=web_folder,
sources=sources,
png_dpi=gallery_cfg.get("png_dpi", GalleryDefaults.png_dpi),
plot_root=gallery_cfg.get("plot_root", GalleryDefaults.plot_root),
cache_enabled=data.get("metadata", {}).get("cache_enabled", GalleryDefaults.cache_enabled),
inherit_from_parent=data.get("metadata", {}).get("inherit_from_parent", GalleryDefaults.inherit_from_parent),
backup_folder=gallery_cfg.get("backup_folder", ""),
)
def to_yaml(self, yaml_file: Union[str, Path]) -> None:
"""Save the current configuration to a YAML file."""
yaml_path = Path(yaml_file)
data = {
"paths": {
"work_dir": str(Path.cwd()),
"web_folder": str(self.web_folder),
},
"gallery": {
"plot_root": self.plot_root,
"png_dpi": self.png_dpi,
"backup_folder": self.backup_folder,
},
"ui": {
"max_recent_plots": 20,
"search_debounce_ms": 300,
},
"metadata": {
"cache_enabled": self.cache_enabled,
"inherit_from_parent": self.inherit_from_parent,
"supported_formats": [".yaml", ".yml", ".json"],
},
"sources": [{"name": s.name, "path": str(s.path)} for s in self.sources],
}
with open(yaml_path, "w") as f:
yaml.dump(data, f, default_flow_style=False)
+14
View File
@@ -0,0 +1,14 @@
gallery:
backup_folder: ''
plot_root: gallery
png_dpi: 400
metadata:
cache_enabled: true
inherit_from_parent: true
paths:
web_folder: ''
work_dir: ''
sources: []
ui:
max_recent_plots: 20
search_debounce_ms: 300
+2 -2
View File
@@ -51,8 +51,8 @@ dev = [
gallery = "gallery.cli:main"
[tool.setuptools]
packages = ["gallery", "gallery.utils"]
package-data = {gallery = ["templates/*", "assets/css/*", "assets/js/*"]}
packages = ["gallery", "gallery.utils", "gallery.config"]
package-data = {gallery = ["templates/*", "assets/css/*", "assets/js/*", "config/*"]}
include-package-data = true
[tool.black]