Unlink assets and put everything into the gallery folder
This commit is contained in:
Kylian Schmidt
+1
-1
@@ -44,6 +44,6 @@ sources:
|
||||
- name: "needle_benchmarks"
|
||||
path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots"
|
||||
- name: "needle_fair_universe"
|
||||
path: "/ceph/kschmidt/needle/plots/fair_universe/"
|
||||
path: "/work/kschmidt/NEEDLE/orchestrator/runs/fair_universe_demo_fixed_normalization/stat_only_histogram_mu_one/plots"
|
||||
- name: "aido_convergence_study"
|
||||
path: "/work/kschmidt/aido/results_convergence/plots/"
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../assets
|
||||
@@ -1,373 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>{{ title }}</title>
|
||||
<link rel="stylesheet" href="{{ assets_path }}/css/main.css">
|
||||
<link rel="stylesheet" href="{{ assets_path }}/css/html-plots.css">
|
||||
|
||||
<!-- MathJax for LaTeX rendering -->
|
||||
<script>
|
||||
MathJax = {
|
||||
tex: {
|
||||
inlineMath: [['$', '$'], ['\\(', '\\)']],
|
||||
displayMath: [['$$', '$$'], ['\\[', '\\]']]
|
||||
}
|
||||
};
|
||||
</script>
|
||||
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Main Content -->
|
||||
<h1>{{ title }}</h1>
|
||||
|
||||
<!-- Search Bar -->
|
||||
<div class="search-container">
|
||||
<input type="text" class="search-box" id="searchBox" placeholder="Search plots..." />
|
||||
<span class="search-icon">🔍</span>
|
||||
<div class="search-results" id="searchResults"></div>
|
||||
</div>
|
||||
|
||||
<!-- Breadcrumb Navigation -->
|
||||
<div class="breadcrumb" id="breadcrumb"></div>
|
||||
|
||||
<!-- Navigation Buttons -->
|
||||
<div class="navigation">
|
||||
<button class="nav-btn" onclick="window.history.back()">
|
||||
← Back
|
||||
</button>
|
||||
{% if relpath != "." %}
|
||||
<a href="../index.html" class="nav-btn">
|
||||
↑ Parent Directory
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Folder Tree -->
|
||||
<div class="folder-tree" id="folderTree"></div>
|
||||
|
||||
<!-- Folder Metadata Section -->
|
||||
{% if folder_metadata %}
|
||||
<div class="metadata-section">
|
||||
<button class="metadata-toggle-btn" onclick="toggleMetadataSection()">
|
||||
<span class="metadata-icon">📋</span>
|
||||
<span class="metadata-label">Folder Information</span>
|
||||
<span class="metadata-arrow" id="metadataArrow">▼</span>
|
||||
</button>
|
||||
|
||||
<div class="metadata-content" id="metadataContent" style="display: none;">
|
||||
<div class="metadata-header">
|
||||
<div class="metadata-file-info">
|
||||
<span class="file-path-label">📁 Metadata file:</span>
|
||||
<code class="file-path" id="metadata-file-path">{{ metadata_file_path }}</code>
|
||||
<button class="copy-path-btn" onclick="copyMetadataPath()" title="Copy path to clipboard">
|
||||
📋 Copy
|
||||
</button>
|
||||
<span class="tip-icon" title="Tip: Create this file in the source directory to add folder-level metadata that will be inherited by all plots in this folder and its subdirectories. Supports both YAML (.yaml/.yml) and JSON (.json) formats.">
|
||||
💡
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metadata-grid">
|
||||
{% for key, value in folder_metadata.items() %}
|
||||
<div class="metadata-item">
|
||||
<span class="metadata-key">{{ key }}:</span>
|
||||
<span class="metadata-value">
|
||||
{% if value is string and (value.startswith('http://') or value.startswith('https://')) %}
|
||||
<a href="{{ value }}" target="_blank" rel="noopener noreferrer">{{ value }}</a>
|
||||
{% elif value is string and '$$' in value %}
|
||||
<span class="latex-content">{{ value }}</span>
|
||||
{% elif value is string and value|length > 100 %}
|
||||
<span class="metadata-long-text">{{ value[:100] }}...</span>
|
||||
<button class="metadata-expand" onclick="expandText(this)">Show more</button>
|
||||
<span class="metadata-full-text" style="display: none;">{{ value }}</span>
|
||||
{% elif value is iterable and value is not string and value is not mapping %}
|
||||
<div class="metadata-yaml-list">
|
||||
{% for item in value %}
|
||||
<div class="yaml-list-item">- {{ item }}</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% elif value is mapping %}
|
||||
<div class="metadata-yaml-object">
|
||||
{% for subkey, subvalue in value.items() %}
|
||||
<div class="yaml-object-item">
|
||||
<span class="yaml-key">{{ subkey }}:</span>
|
||||
{% if subvalue is iterable and subvalue is not string and subvalue is not mapping %}
|
||||
<div class="yaml-nested-list">
|
||||
{% for nested_item in subvalue %}
|
||||
<div class="yaml-nested-item">- {{ nested_item }}</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% elif subvalue is mapping %}
|
||||
<div class="yaml-nested-object">
|
||||
{% for nested_key, nested_value in subvalue.items() %}
|
||||
<div class="yaml-nested-item">{{ nested_key }}: {{ nested_value }}</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
<span class="yaml-value"> {{ subvalue }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% else %}
|
||||
{{ value }}
|
||||
{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- View Toggle Controls - Only show if there are plot items -->
|
||||
{% if items %}
|
||||
<div class="controls-container">
|
||||
<div class="sort-controls">
|
||||
<label class="sort-label">Sort by:</label>
|
||||
<button class="sort-btn active" data-sort="name" title="Sort by Name">
|
||||
📝 Name
|
||||
</button>
|
||||
<button class="sort-btn" data-sort="time" title="Sort by Creation Time">
|
||||
🕒 Time
|
||||
</button>
|
||||
<button class="sort-order-btn" data-order="asc" title="Sort Order">
|
||||
↑
|
||||
</button>
|
||||
</div>
|
||||
<div class="view-controls">
|
||||
<button class="view-btn active" data-view="grid" title="Grid View">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16">
|
||||
<rect x="1" y="1" width="6" height="6" fill="currentColor"/>
|
||||
<rect x="9" y="1" width="6" height="6" fill="currentColor"/>
|
||||
<rect x="1" y="9" width="6" height="6" fill="currentColor"/>
|
||||
<rect x="9" y="9" width="6" height="6" fill="currentColor"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="view-btn" data-view="list-large" title="Large List View">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16">
|
||||
<rect x="1" y="2" width="4" height="3" fill="currentColor"/>
|
||||
<rect x="7" y="2" width="8" height="1" fill="currentColor"/>
|
||||
<rect x="7" y="4" width="6" height="1" fill="currentColor"/>
|
||||
<rect x="1" y="7" width="4" height="3" fill="currentColor"/>
|
||||
<rect x="7" y="7" width="8" height="1" fill="currentColor"/>
|
||||
<rect x="7" y="9" width="6" height="1" fill="currentColor"/>
|
||||
<rect x="1" y="12" width="4" height="3" fill="currentColor"/>
|
||||
<rect x="7" y="12" width="8" height="1" fill="currentColor"/>
|
||||
<rect x="7" y="14" width="6" height="1" fill="currentColor"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="view-btn" data-view="list-compact" title="Compact List View">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16">
|
||||
<rect x="1" y="3" width="14" height="1" fill="currentColor"/>
|
||||
<rect x="1" y="6" width="14" height="1" fill="currentColor"/>
|
||||
<rect x="1" y="9" width="14" height="1" fill="currentColor"/>
|
||||
<rect x="1" y="12" width="14" height="1" fill="currentColor"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Plot Container -->
|
||||
<div class="plot-container grid-view" id="plotContainer">
|
||||
{% for item in items %}
|
||||
<div class="plot-item grid-item {% if item.is_html %}html-plot{% endif %}"
|
||||
data-name="{{ item.name }}"
|
||||
data-time="{{ item.creation_time|default(0) }}">
|
||||
{% if item.is_html %}
|
||||
<a href="{{ item.html_href }}" class="plot-link" target="_blank">
|
||||
<div class="html-thumbnail">
|
||||
<div class="html-indicator">HTML</div>
|
||||
<div class="html-preview">Click to open interactive plot</div>
|
||||
</div>
|
||||
</a>
|
||||
{% else %}
|
||||
<a href="{{ item.pdf_href }}" class="plot-link">
|
||||
<img src="{{ item.png_href }}" alt="{{ item.name }}" class="plot-thumbnail">
|
||||
</a>
|
||||
{% endif %}
|
||||
<div class="plot-info">
|
||||
<div class="plot-name" title="{{ item.name }}">{{ item.name }}</div>
|
||||
<div class="plot-date" title="Created: {{ item.creation_time|default(0)|int|datetime_from_timestamp|strftime('%Y-%m-%d %H:%M') if item.creation_time and item.creation_time|int > 0 else 'Unknown' }}">
|
||||
{% if item.creation_time and item.creation_time|int > 0 %}
|
||||
{{ item.creation_time|int|datetime_from_timestamp|strftime('%Y-%m-%d') }}
|
||||
{% else %}
|
||||
Unknown
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Subdirectories -->
|
||||
{% if subdirs %}
|
||||
<h2>Subdirectories</h2>
|
||||
<ul>
|
||||
{% for sub in subdirs %}
|
||||
<li><a href="{{ sub }}/index.html">📁 {{ sub }}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
|
||||
<!-- Recent Plots Sidebar -->
|
||||
<div class="sidebar-overlay" id="sidebarOverlay" onclick="toggleSidebar()"></div>
|
||||
<div class="sidebar" id="sidebar">
|
||||
<div class="sidebar-header">
|
||||
<h3 class="sidebar-title">Recent Plots</h3>
|
||||
<button class="sidebar-close" onclick="toggleSidebar()">×</button>
|
||||
</div>
|
||||
<div class="sidebar-content" id="sidebarContent">
|
||||
<div style="text-align: center; color: var(--breadcrumb-color); margin: 2rem 0;">
|
||||
No recent plots yet
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Floating Action Buttons -->
|
||||
<div class="floating-buttons">
|
||||
<button class="floating-btn sidebar-toggle" onclick="toggleSidebar()" id="sidebarToggle" title="Recent Plots (Ctrl+R)">
|
||||
📋
|
||||
</button>
|
||||
<button class="floating-btn compare-toggle" onclick="app.toggleCompareMode()" id="compareToggle" title="Compare Plots (Ctrl+C)">
|
||||
⚖️
|
||||
</button>
|
||||
<button class="floating-btn theme-toggle" onclick="toggleTheme()" id="themeToggle" title="Toggle Theme (Ctrl+T)">
|
||||
☀️
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Keyboard Shortcuts Help -->
|
||||
<div class="shortcuts-help" id="shortcutsHelp">
|
||||
<h4>Keyboard Shortcuts</h4>
|
||||
<div class="shortcut-item">
|
||||
<span>Search</span>
|
||||
<span class="shortcut-key">Ctrl+K</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Recent plots</span>
|
||||
<span class="shortcut-key">Ctrl+R</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Compare plots</span>
|
||||
<span class="shortcut-key">Ctrl+C</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Export plots</span>
|
||||
<span class="shortcut-key">Ctrl+E</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Exit selection mode</span>
|
||||
<span class="shortcut-key">Esc</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Toggle theme</span>
|
||||
<span class="shortcut-key">Ctrl+T</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Toggle view</span>
|
||||
<span class="shortcut-key">Ctrl+V</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Sort by name</span>
|
||||
<span class="shortcut-key">Ctrl+N</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Sort by time</span>
|
||||
<span class="shortcut-key">Ctrl+M</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Toggle sort order</span>
|
||||
<span class="shortcut-key">Ctrl+O</span>
|
||||
</div>
|
||||
<div class="shortcut-item">
|
||||
<span>Help</span>
|
||||
<span class="shortcut-key">?</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Gallery Statistics -->
|
||||
<div class="gallery-stats" id="galleryStats">
|
||||
<div class="stats-item">
|
||||
<span class="stats-label">📊 Files:</span>
|
||||
<span class="stats-value" id="fileCount">0</span>
|
||||
</div>
|
||||
<div class="stats-item">
|
||||
<span class="stats-label">📁 Folders:</span>
|
||||
<span class="stats-value" id="folderCount">0</span>
|
||||
</div>
|
||||
<div class="stats-item">
|
||||
<span class="stats-label">💾 Size:</span>
|
||||
<span class="stats-value" id="totalSize">0 KB</span>
|
||||
</div>
|
||||
<div class="stats-item">
|
||||
<span class="stats-label">🕒 Updated:</span>
|
||||
<span class="stats-value" id="lastUpdated">Now</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Plot Comparison Overlay -->
|
||||
<div class="comparison-overlay" id="comparisonOverlay">
|
||||
<div class="comparison-container">
|
||||
<div class="comparison-header">
|
||||
<h2 class="comparison-title">Plot Comparison</h2>
|
||||
<button class="comparison-close" onclick="app.closeComparison()" title="Close Comparison (Esc)">×</button>
|
||||
</div>
|
||||
<div class="comparison-content">
|
||||
<div class="comparison-panel">
|
||||
<div class="comparison-panel-header">
|
||||
<span class="comparison-panel-title" id="leftPlotTitle">Plot A</span>
|
||||
<button class="comparison-replace-btn" onclick="app.replacePlot('left')" id="leftReplaceBtn">Replace</button>
|
||||
</div>
|
||||
<div class="comparison-panel-content">
|
||||
<div class="comparison-plot-container" id="leftPlotContainer">
|
||||
<div class="comparison-placeholder" onclick="app.selectPlotForComparison('left')">
|
||||
📊 Click to select first plot
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="comparison-panel">
|
||||
<div class="comparison-panel-header">
|
||||
<span class="comparison-panel-title" id="rightPlotTitle">Plot B</span>
|
||||
<button class="comparison-replace-btn" onclick="app.replacePlot('right')" id="rightReplaceBtn">Replace</button>
|
||||
</div>
|
||||
<div class="comparison-panel-content">
|
||||
<div class="comparison-plot-container" id="rightPlotContainer">
|
||||
<div class="comparison-placeholder" onclick="app.selectPlotForComparison('right')">
|
||||
📊 Click to select second plot
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Configuration for JavaScript -->
|
||||
<script>
|
||||
// Configuration object for the gallery app
|
||||
window.galleryConfig = {
|
||||
searchDebounceMs: {{ ui.search_debounce_ms|default(300) }},
|
||||
maxRecentPlots: {{ ui.max_recent_plots|default(20) }},
|
||||
workDir: "{{ paths.work_dir }}",
|
||||
stats: {% if stats %}{{ stats|tojson }}{% else %}null{% endif %}
|
||||
};
|
||||
</script>
|
||||
|
||||
<!-- Metadata Popup Script -->
|
||||
<script src="{{ assets_path }}/js/metadata-popup.js"></script>
|
||||
|
||||
<!-- Metadata Section Script -->
|
||||
<script src="{{ assets_path }}/js/metadata-section.js"></script>
|
||||
|
||||
<!-- Folder Metadata Script -->
|
||||
<script src="{{ assets_path }}/js/folder-metadata.js"></script>
|
||||
|
||||
<!-- Main JavaScript Application -->
|
||||
<script type="module" src="{{ assets_path }}/js/main.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,30 +0,0 @@
|
||||
import zipfile
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
|
||||
WEB_FOLDER = Path("plots")
|
||||
BACKUP_FOLDER = Path("backups")
|
||||
|
||||
|
||||
def create_backup():
|
||||
"""Create a backup of the web folder."""
|
||||
today = datetime.date.today().strftime("%Y%m%d")
|
||||
backup_name = f"backup-{today}.zip"
|
||||
backup_path = BACKUP_FOLDER / backup_name
|
||||
|
||||
BACKUP_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if backup_path.exists():
|
||||
print(f"Backup already exists: {backup_path}")
|
||||
else:
|
||||
print(f"Creating backup: {backup_path}")
|
||||
with zipfile.ZipFile(backup_path, "w", zipfile.ZIP_DEFLATED) as zipf:
|
||||
for path in WEB_FOLDER.rglob("*"):
|
||||
if path.is_file():
|
||||
arcname = path.relative_to(WEB_FOLDER.parent)
|
||||
zipf.write(path, arcname)
|
||||
print("✅ Backup complete.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
create_backup()
|
||||
-145
@@ -1,145 +0,0 @@
|
||||
"""
|
||||
Scientific Gallery Configuration Management
|
||||
|
||||
This module provides dataclasses and utilities for managing configuration
|
||||
of the scientific gallery system, including paths, gallery settings,
|
||||
UI preferences, and data sources.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class PathConfig:
|
||||
"""Configuration for system paths and directories."""
|
||||
work_dir: str
|
||||
web_folder: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class GalleryConfig:
|
||||
"""Configuration for gallery generation and display settings."""
|
||||
plot_root: str
|
||||
png_dpi: int
|
||||
backup_folder: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class UIConfig:
|
||||
"""Configuration for user interface behavior and preferences."""
|
||||
max_recent_plots: int
|
||||
search_debounce_ms: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetadataConfig:
|
||||
"""Configuration for metadata handling."""
|
||||
cache_enabled: bool = True
|
||||
inherit_from_parent: bool = True
|
||||
supported_formats: list[str] = field(
|
||||
default_factory=lambda: ['.yaml', '.yml', '.json']
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GalleryItem:
|
||||
"""Represents a single data source for the gallery."""
|
||||
name: str
|
||||
path: Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""
|
||||
Main configuration class that aggregates all gallery settings.
|
||||
|
||||
Provides backward compatibility properties and methods for loading
|
||||
configuration from YAML files.
|
||||
"""
|
||||
paths: PathConfig
|
||||
gallery: GalleryConfig
|
||||
ui: UIConfig
|
||||
metadata: MetadataConfig
|
||||
sources: list[GalleryItem] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def web_folder(self):
|
||||
"""Backward compatibility property for web folder path."""
|
||||
return self.paths.web_folder
|
||||
|
||||
@property
|
||||
def png_dpi(self):
|
||||
"""Backward compatibility property for PNG conversion DPI."""
|
||||
return self.gallery.png_dpi
|
||||
|
||||
@property
|
||||
def plot_root(self):
|
||||
"""Backward compatibility property for plot root directory."""
|
||||
return self.gallery.plot_root
|
||||
|
||||
@property
|
||||
def backup_folder(self):
|
||||
"""Backward compatibility property for backup folder path."""
|
||||
return self.gallery.backup_folder
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, yaml_file: str) -> "Config":
|
||||
"""
|
||||
Load configuration from a YAML file.
|
||||
|
||||
Args:
|
||||
yaml_file: Path to the YAML configuration file
|
||||
|
||||
Returns:
|
||||
Config instance with loaded settings
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the YAML file doesn't exist
|
||||
yaml.YAMLError: If the YAML file is malformed
|
||||
"""
|
||||
with open(yaml_file, "r") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
paths_data = data.get('paths', {})
|
||||
gallery_data = data.get('gallery', {})
|
||||
ui_data = data.get('ui', {})
|
||||
metadata_data = data.get('metadata', {})
|
||||
sources_data = data.get('sources', [])
|
||||
|
||||
paths = PathConfig(**paths_data)
|
||||
gallery = GalleryConfig(**gallery_data)
|
||||
ui = UIConfig(**ui_data)
|
||||
metadata = MetadataConfig(**metadata_data)
|
||||
|
||||
sources = [
|
||||
GalleryItem(name=source["name"], path=Path(source["path"]))
|
||||
for source in sources_data
|
||||
]
|
||||
|
||||
return cls(
|
||||
paths=paths,
|
||||
gallery=gallery,
|
||||
ui=ui,
|
||||
metadata=metadata,
|
||||
sources=sources
|
||||
)
|
||||
|
||||
def to_yaml(self, yaml_file: str) -> None:
|
||||
"""
|
||||
Save the current configuration to a YAML file.
|
||||
|
||||
Args:
|
||||
yaml_file: Path where to save the YAML configuration
|
||||
|
||||
Raises:
|
||||
IOError: If unable to write to the specified file
|
||||
"""
|
||||
with open(yaml_file, "w") as f:
|
||||
yaml.dump(asdict(self), f, default_flow_style=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
config = Config.from_yaml("config.yaml")
|
||||
print("Loaded config successfully:", config)
|
||||
@@ -1,159 +0,0 @@
|
||||
"""
|
||||
Metadata Management for Scientific Gallery Generator
|
||||
|
||||
This module handles loading, parsing, and caching of metadata for plots
|
||||
and folders in the gallery system. Supports YAML and JSON formats with
|
||||
hierarchical inheritance.
|
||||
|
||||
Features:
|
||||
- Load metadata from YAML/JSON files
|
||||
- Hierarchical metadata inheritance from parent folders
|
||||
- Plot-specific metadata overrides
|
||||
- Metadata caching for performance
|
||||
"""
|
||||
|
||||
import json
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
def load_metadata_file(metadata_path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Load metadata from a YAML or JSON file.
|
||||
|
||||
Args:
|
||||
metadata_path: Path to the metadata file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the metadata, empty dict if file doesn't exist
|
||||
or can't be parsed
|
||||
"""
|
||||
if not metadata_path.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
with metadata_path.open('r', encoding='utf-8') as f:
|
||||
suffix_lower = metadata_path.suffix.lower()
|
||||
if suffix_lower == '.yaml' or suffix_lower == '.yml':
|
||||
return yaml.safe_load(f) or {}
|
||||
elif metadata_path.suffix.lower() == '.json':
|
||||
return json.load(f) or {}
|
||||
else:
|
||||
print(f"Warning: Unknown metadata file format: "
|
||||
f"{metadata_path}")
|
||||
return {}
|
||||
except (yaml.YAMLError, json.JSONDecodeError, IOError) as e:
|
||||
print(f"Warning: Could not parse metadata file {metadata_path}: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def load_folder_metadata(folder_path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
Load folder-level metadata from metadata.yaml, metadata.yml, or metadata.json.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the folder to check for metadata
|
||||
|
||||
Returns:
|
||||
Dictionary containing the folder metadata
|
||||
"""
|
||||
# Try YAML first, then JSON for backwards compatibility
|
||||
for filename in ['metadata.yaml', 'metadata.yml', 'metadata.json']:
|
||||
metadata_path = folder_path / filename
|
||||
if metadata_path.exists():
|
||||
return load_metadata_file(metadata_path)
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
def get_metadata_file_path(folder_path: Path) -> str:
|
||||
"""
|
||||
Get the metadata file path for a folder. Returns existing file if found,
|
||||
otherwise suggests metadata.yaml (preferred format).
|
||||
|
||||
Args:
|
||||
folder_path: Path to the folder to check for metadata
|
||||
|
||||
Returns:
|
||||
String path to the metadata file (existing or suggested)
|
||||
"""
|
||||
# Preferred order: YAML first, then JSON
|
||||
preferred_files = ['metadata.yaml', 'metadata.yml', 'metadata.json']
|
||||
|
||||
for filename in preferred_files:
|
||||
metadata_path = folder_path / filename
|
||||
if metadata_path.exists():
|
||||
return str(metadata_path)
|
||||
|
||||
# If no file exists, suggest metadata.yaml (preferred format)
|
||||
return str(folder_path / 'metadata.yaml')
|
||||
|
||||
|
||||
def merge_metadata(
|
||||
parent_metadata: Dict[str, Any],
|
||||
child_metadata: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Merge parent and child metadata, with child values overriding parent
|
||||
values.
|
||||
|
||||
Args:
|
||||
parent_metadata: Metadata from parent folder
|
||||
child_metadata: Metadata from current folder
|
||||
|
||||
Returns:
|
||||
Merged metadata dictionary
|
||||
"""
|
||||
merged = parent_metadata.copy()
|
||||
merged.update(child_metadata)
|
||||
return merged
|
||||
|
||||
|
||||
def resolve_metadata_for_plot(
|
||||
plot_path: Path,
|
||||
inherited_metadata: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Resolve metadata for a specific plot by merging inherited metadata
|
||||
with plot-specific metadata.
|
||||
|
||||
Args:
|
||||
plot_path: Path to the plot file (PDF)
|
||||
inherited_metadata: Metadata inherited from folder hierarchy
|
||||
|
||||
Returns:
|
||||
Final merged metadata for the plot
|
||||
"""
|
||||
plot_stem = plot_path.stem
|
||||
plot_dir = plot_path.parent
|
||||
|
||||
# Check for plot-specific metadata files
|
||||
for suffix in ['.yaml', '.yml', '.json']:
|
||||
plot_metadata_path = plot_dir / f"{plot_stem}{suffix}"
|
||||
if plot_metadata_path.exists():
|
||||
plot_metadata = load_metadata_file(plot_metadata_path)
|
||||
return merge_metadata(inherited_metadata, plot_metadata)
|
||||
|
||||
# No plot-specific metadata found, return inherited metadata
|
||||
return inherited_metadata.copy()
|
||||
|
||||
|
||||
def save_metadata_cache(
|
||||
web_dir: Path,
|
||||
plot_metadata_cache: Dict[str, Dict[str, Any]]
|
||||
) -> None:
|
||||
"""
|
||||
Save plot metadata cache to meta_cache.json in the web directory.
|
||||
|
||||
Args:
|
||||
web_dir: Web directory where the cache file should be saved
|
||||
plot_metadata_cache: Dictionary mapping plot names to their metadata
|
||||
"""
|
||||
cache_path = web_dir / "meta_cache.json"
|
||||
try:
|
||||
with cache_path.open('w', encoding='utf-8') as f:
|
||||
json.dump(plot_metadata_cache, f, indent=2, ensure_ascii=False)
|
||||
print(f"Saved metadata cache: {cache_path}")
|
||||
except IOError as e:
|
||||
print(f"Warning: Could not save metadata cache {cache_path}: {e}")
|
||||
@@ -1,230 +0,0 @@
|
||||
import shutil
|
||||
from typing import Any, Dict
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from jinja2 import Template
|
||||
|
||||
from utils.metadata import (
|
||||
resolve_metadata_for_plot,
|
||||
get_metadata_file_path,
|
||||
)
|
||||
from utils.stats import (
|
||||
calculate_directory_stats,
|
||||
format_file_size,
|
||||
)
|
||||
from utils.config import Config
|
||||
|
||||
|
||||
def process_html_file(
|
||||
html_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None
|
||||
) -> dict:
|
||||
"""
|
||||
Process HTML plot file, copying it to web directory.
|
||||
|
||||
Args:
|
||||
html_file: Path to the source HTML file
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
web_html = web_dir / html_file.name
|
||||
|
||||
if needs_update(html_file, web_html):
|
||||
shutil.copy2(html_file, web_html)
|
||||
else:
|
||||
print(f"Skipping {html_file.name} (up to date)")
|
||||
|
||||
# Get source file creation time
|
||||
source_creation_time = int(html_file.stat().st_ctime)
|
||||
|
||||
# Resolve metadata if provided
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(html_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": html_file.stem,
|
||||
"html_href": html_file.name,
|
||||
"is_html": True,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def process_plot_files(
|
||||
CONFIG: Config,
|
||||
plot_file: Path,
|
||||
web_dir: Path,
|
||||
current_metadata: Dict[str, Any] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Process plot files (PDF/PNG or HTML), handling conversion and copying.
|
||||
|
||||
Args:
|
||||
plot_file: Path to the source plot file (PDF or HTML)
|
||||
web_dir: Target web directory
|
||||
current_metadata: Current metadata dictionary for the plot
|
||||
|
||||
Returns:
|
||||
Dictionary containing plot information
|
||||
"""
|
||||
if plot_file.suffix.lower() == '.html':
|
||||
return process_html_file(plot_file, web_dir, current_metadata)
|
||||
|
||||
# Handle PDF files
|
||||
png_file = plot_file.with_suffix(".png")
|
||||
web_pdf = web_dir / plot_file.name
|
||||
web_png = web_dir / png_file.name
|
||||
|
||||
if needs_update(plot_file, web_pdf):
|
||||
shutil.copy2(plot_file, web_pdf)
|
||||
else:
|
||||
print(f"Skipping {plot_file.name} (up to date)")
|
||||
|
||||
if not png_file.exists():
|
||||
convert_pdf_to_png(plot_file, CONFIG=CONFIG)
|
||||
|
||||
if needs_update(png_file, web_png):
|
||||
shutil.copy2(png_file, web_png)
|
||||
else:
|
||||
print(f"Skipping {png_file.name} (up to date)")
|
||||
|
||||
source_creation_time = int(plot_file.stat().st_ctime)
|
||||
|
||||
plot_metadata = {}
|
||||
if current_metadata is not None:
|
||||
plot_metadata = resolve_metadata_for_plot(plot_file, current_metadata)
|
||||
|
||||
return {
|
||||
"name": plot_file.stem,
|
||||
"pdf_href": plot_file.name,
|
||||
"png_href": png_file.name,
|
||||
"is_html": False,
|
||||
"metadata": plot_metadata,
|
||||
"creation_time": source_creation_time
|
||||
}
|
||||
|
||||
|
||||
def render_gallery_page(
|
||||
CONFIG: Config,
|
||||
template: Template,
|
||||
web_dir: Path,
|
||||
items: list,
|
||||
subdirs: list,
|
||||
relative_path: Path,
|
||||
title: str = None,
|
||||
metadata: dict = None
|
||||
) -> None:
|
||||
"""
|
||||
Unified template rendering for all gallery pages.
|
||||
|
||||
Args:
|
||||
web_dir: Target web directory
|
||||
items: List of plot items
|
||||
subdirs: List of subdirectory names
|
||||
relative_path: Relative path from gallery root
|
||||
title: Page title (optional)
|
||||
metadata: Metadata dictionary (optional)
|
||||
"""
|
||||
if title is None:
|
||||
title = "Gallery" if relative_path == Path(
|
||||
".") else f"Gallery: {relative_path}"
|
||||
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
# Calculate statistics
|
||||
current_stats = calculate_directory_stats(web_dir)
|
||||
stats = {
|
||||
"file_count": len(items),
|
||||
"folder_count": len(subdirs),
|
||||
"total_size": format_file_size(current_stats["total_size"]),
|
||||
"total_size_bytes": current_stats["total_size"]
|
||||
}
|
||||
|
||||
# Calculate relative path to assets
|
||||
if relative_path == Path("."):
|
||||
assets_path = "../assets"
|
||||
else:
|
||||
depth = len(relative_path.parts)
|
||||
assets_path = "../" * (depth + 1) + "assets"
|
||||
|
||||
# For root level, show only directory structure
|
||||
if relative_path == Path("."):
|
||||
items = []
|
||||
|
||||
output_html = web_dir / "index.html"
|
||||
with output_html.open("w") as f:
|
||||
rendered_html = template.render(
|
||||
title=title,
|
||||
items=items,
|
||||
subdirs=subdirs,
|
||||
relpath=str(relative_path),
|
||||
paths=CONFIG.paths,
|
||||
ui=CONFIG.ui,
|
||||
stats=stats,
|
||||
folder_metadata=metadata,
|
||||
assets_path=assets_path,
|
||||
source_dir=str(web_dir),
|
||||
metadata_file_path=get_metadata_file_path(web_dir)
|
||||
)
|
||||
f.write(rendered_html)
|
||||
|
||||
print(f"Generated {output_html}")
|
||||
|
||||
|
||||
def convert_pdf_to_png(pdf_path: Path, CONFIG: Config) -> None:
|
||||
"""
|
||||
Convert a PDF file to PNG format using ImageMagick.
|
||||
|
||||
Only converts if the PNG doesn't exist or if the PDF is newer than
|
||||
the PNG (with a 30-second buffer to handle filesystem timing issues).
|
||||
|
||||
Args:
|
||||
pdf_path: Path to the source PDF file
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If ImageMagick conversion fails
|
||||
"""
|
||||
png_path = pdf_path.with_suffix(".png")
|
||||
|
||||
if png_path.exists():
|
||||
pdf_mtime = pdf_path.stat().st_mtime
|
||||
png_mtime = png_path.stat().st_mtime
|
||||
if png_mtime >= (pdf_mtime + 30):
|
||||
return
|
||||
else:
|
||||
print(f"PDF {pdf_path.name} is newer than PNG, reconverting...")
|
||||
|
||||
print(f"Converting {pdf_path} to PNG")
|
||||
subprocess.run([
|
||||
"convert",
|
||||
"-density", str(CONFIG.png_dpi),
|
||||
str(pdf_path),
|
||||
"-quality", "95",
|
||||
str(png_path)
|
||||
], check=True)
|
||||
|
||||
|
||||
def needs_update(source_file: Path, target_file: Path) -> bool:
|
||||
"""
|
||||
Check if target file needs updating based on source modification time.
|
||||
|
||||
Args:
|
||||
source_file: Path to the source file
|
||||
target_file: Path to the target file
|
||||
|
||||
Returns:
|
||||
True if target needs update, False otherwise
|
||||
"""
|
||||
if not target_file.exists():
|
||||
return True
|
||||
|
||||
source_mtime = source_file.stat().st_mtime
|
||||
target_mtime = target_file.stat().st_mtime
|
||||
|
||||
return source_mtime > (target_mtime + 30)
|
||||
@@ -1,62 +0,0 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def calculate_directory_stats(directory: Path) -> dict:
|
||||
"""
|
||||
Calculate statistics for a directory.
|
||||
|
||||
Args:
|
||||
directory: Path to the directory to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary containing file count, folder count, and total size
|
||||
"""
|
||||
stats = {
|
||||
"file_count": 0,
|
||||
"folder_count": 0,
|
||||
"total_size": 0,
|
||||
"pdf_size": 0,
|
||||
"png_size": 0,
|
||||
}
|
||||
|
||||
if not directory.exists():
|
||||
return stats
|
||||
|
||||
for item in directory.rglob("*"):
|
||||
if item.is_file():
|
||||
stats["file_count"] += 1
|
||||
size = item.stat().st_size
|
||||
stats["total_size"] += size
|
||||
|
||||
if item.suffix.lower() == '.pdf':
|
||||
stats["pdf_size"] += size
|
||||
elif item.suffix.lower() == '.png':
|
||||
stats["png_size"] += size
|
||||
elif item.is_dir():
|
||||
stats["folder_count"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
Format file size in human readable format.
|
||||
|
||||
Args:
|
||||
size_bytes: Size in bytes
|
||||
|
||||
Returns:
|
||||
Formatted size string
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0 B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB", "TB"]
|
||||
size = float(size_bytes)
|
||||
i = 0
|
||||
while size >= 1024 and i < len(size_names) - 1:
|
||||
size /= 1024
|
||||
i += 1
|
||||
|
||||
return f"{size:.1f} {size_names[i]}"
|
||||
|
||||
Reference in New Issue
Block a user