lars/ETPlot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add pyproject.toml

Browse Source
This commit is contained in:
Kylian Schmidt
2025-07-22 09:49:43 +02:00
parent 491a4a9fd2
commit 6eeacdc9b7
5 changed files with 114 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitlab-ci.yml
-31
View File
@@ -1,31 +0,0 @@
# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/user/application_security/secret_detection/pipeline/configure
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
stages:
- build
- test
- deploy
- review
- dast
- staging
- canary
- production
- incremental rollout 10%
- incremental rollout 25%
- incremental rollout 50%
- incremental rollout 100%
- performance
- cleanup
- secret-detection
sast:
stage: test
include:
- template: Auto-DevOps.gitlab-ci.yml
variables:
SECRET_DETECTION_ENABLED: 'true'
secret_detection:
stage: secret-detection
config.yaml
+1 -1
View File
@@ -47,5 +47,5 @@ metadata:
sources:
- name: "ttbar_analysis"
path: "/work/kschmidt/NEEDLE/test_analysis/data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/"
- name: "NEEDLE_benchmarks"
- name: "needle_benchmarks"
path: "/work/kschmidt/NEEDLE/orchestrator/ml/benchmarks/plots"
debug_template.py
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
# Test template rendering
env = Environment(loader=FileSystemLoader("."))
template = env.get_template("templates/gallery.html")
# Mock data
test_items = [{
"name": "test_plot",
"pdf_href": "test_plot.pdf",
"png_href": "test_plot.png",
"metadata": {"key1": "value1", "key2": "value2"}
}]
test_config = {
"paths": {"work_dir": "/work/kschmidt/web"},
"ui": {"search_debounce_ms": 300, "max_recent_plots": 20}
}
rendered = template.render(
title="Test Gallery",
items=test_items,
subdirs=["test_subdir"],
relpath="test/path",
paths=test_config["paths"],
ui=test_config["ui"],
stats={"file_count": 1, "folder_count": 1, "total_size": "100 KB", "total_size_bytes": 100000},
folder_metadata={},
assets_path="../assets"
)
# Extract just the metadata button part
lines = rendered.split('\n')
for i, line in enumerate(lines):
if 'metadata-btn' in line:
print(f"Line {i}: {line.strip()}")
if i > 0:
print(f"Line {i-1}: {lines[i-1].strip()}")
if i < len(lines)-1:
print(f"Line {i+1}: {lines[i+1].strip()}")
break
print("\n--- Assets path in template ---")
for i, line in enumerate(lines):
if 'assets_path' in line:
print(f"Line {i}: {line.strip()}")
generate_gallery.py
+35 -166
View File
@@ -15,6 +15,8 @@ Features:
import subprocess
import shutil
import os
import sys
from pathlib import Path
from typing import Dict, Any, Optional
from jinja2 import Environment, FileSystemLoader
@@ -95,8 +97,7 @@ def build_gallery(source_dir: Path, web_dir: Path,
Processes all PDF files in the source directory, converts them to PNG,
copies both to the web directory, and generates index.html files with
navigation and thumbnails. HTML files are always regenerated to ensure
that new subdirectories appear in navigation immediately.
navigation and thumbnails. Now includes metadata support.
Args:
source_dir: Source directory containing PDF files
@@ -115,15 +116,12 @@ def build_gallery(source_dir: Path, web_dir: Path,
current_metadata = merge_metadata(inherited_metadata, folder_metadata)
pdf_files = list(source_dir.glob("*.pdf"))
png_files = list(source_dir.glob("*.png"))
subdirs = [d for d in source_dir.iterdir() if d.is_dir()]
items = []
plot_metadata_cache = {}
processed_stems = set()
# First, process PDF files (with PNG conversion)
for pdf_file in pdf_files:
png_file = pdf_file.with_suffix(".png")
@@ -147,12 +145,6 @@ def build_gallery(source_dir: Path, web_dir: Path,
# Resolve metadata for this specific plot
plot_metadata = resolve_metadata_for_plot(pdf_file, current_metadata)
if not plot_metadata:
plot_metadata = {}
for k, v in plot_metadata.items():
if isinstance(v, str):
plot_metadata[k] = (v.replace('<', '').replace('>', '')
.replace('&', '').replace('"', "'"))
plot_metadata_cache[pdf_file.stem] = plot_metadata
items.append({
@@ -161,40 +153,6 @@ def build_gallery(source_dir: Path, web_dir: Path,
"png_href": png_file.name,
"metadata": plot_metadata
})
processed_stems.add(pdf_file.stem)
# Second, process standalone PNG files (no corresponding PDF)
for png_file in png_files:
# Skip if we already processed this PNG via its PDF
if png_file.stem in processed_stems:
continue
web_png = web_dir / png_file.name
if needs_update(png_file, web_png):
print(f"Copying {png_file} to {web_png}")
shutil.copy2(png_file, web_png)
else:
print(f"Skipping {png_file.name} (up to date)")
# Resolve metadata for this standalone PNG plot
plot_metadata = resolve_metadata_for_plot(png_file, current_metadata)
if not plot_metadata:
plot_metadata = {}
for k, v in plot_metadata.items():
if isinstance(v, str):
plot_metadata[k] = (v.replace('<', '').replace('>', '')
.replace('&', '').replace('"', "'"))
plot_metadata_cache[png_file.stem] = plot_metadata
# For standalone PNGs, link to the PNG for both thumbnail and main view
items.append({
"name": png_file.stem,
"pdf_href": png_file.name, # Link to PNG instead of PDF
"png_href": png_file.name, # Thumbnail is also the PNG
"metadata": plot_metadata
})
processed_stems.add(png_file.stem)
# Save metadata cache for this directory
save_metadata_cache(web_dir, plot_metadata_cache)
@@ -210,6 +168,17 @@ def build_gallery(source_dir: Path, web_dir: Path,
output_html = web_dir / "index.html"
# Always regenerate HTML to ensure subdirectory changes are reflected
# This ensures new subdirectories appear in navigation
force_regeneration = True
if output_html.exists() and not force_regeneration:
html_mtime = output_html.stat().st_mtime
# Check if any subdirectory is newer than the HTML file
for subdir in subdirs:
if subdir.stat().st_mtime > html_mtime:
print(f"Subdirectory {subdir.name} is newer, forcing regeneration")
break
if relative_path == Path("."):
title = "Gallery"
else:
@@ -309,62 +278,19 @@ def format_file_size(size_bytes: int) -> str:
return f"{size:.1f} {size_names[i]}"
def generate_root_index(gallery_root: Path) -> None:
def refresh_gallery_cgi():
"""
Generate the main index.html file for the gallery root.
This serves as the entry point to navigate to all configured sources.
Args:
gallery_root: Path to the gallery root directory
CGI handler to refresh the gallery from a web request.
Outputs a minimal HTTP response and triggers gallery regeneration.
"""
# Collect information about all source directories
subdirs = []
total_plots = 0
total_size = 0
for source in CONFIG.sources:
source_web_dir = gallery_root / source.name
if source_web_dir.exists():
subdirs.append(source.name)
# Count plots and calculate size for this source
stats = calculate_directory_stats(source_web_dir)
total_plots += stats.get("file_count", 0)
total_size += stats.get("total_size", 0)
# Calculate overall statistics
stats = {
"file_count": total_plots,
"folder_count": len(subdirs),
"total_size": format_file_size(total_size),
"total_size_bytes": total_size
}
# Assets are at the same level as gallery for root
assets_path = "../assets"
output_html = gallery_root / "index.html"
with output_html.open("w") as f:
rendered_html = template.render(
title="Scientific Gallery",
items=[], # No individual plots at root level
subdirs=subdirs,
relpath=".",
paths=CONFIG.paths,
ui=CONFIG.ui,
stats=stats,
folder_metadata={
"description": ("Main gallery containing scientific plots "
"and analyses"),
"sources": [{"name": s.name, "path": s.path}
for s in CONFIG.sources]
},
assets_path=assets_path
)
f.write(rendered_html)
print(f"Generated root gallery index: {output_html}")
import traceback
print("Content-Type: text/plain\n")
try:
main()
print("Gallery refreshed successfully.")
except Exception as e:
print(f"Error refreshing gallery: {e}")
traceback.print_exc(file=sys.stdout)
def main(clean_first: bool = False) -> None:
@@ -478,65 +404,6 @@ def main(clean_first: bool = False) -> None:
print(f"Generated {output_html}")
print(f"Processed {source.name}: {source.path}")
elif source_path.is_file() and source_path.suffix == '.png':
# Handle standalone PNG files
source_web_dir = gallery_root / source.name
source_web_dir.mkdir(parents=True, exist_ok=True)
png_name = source_path.name
web_png_path = source_web_dir / png_name
if needs_update(source_path, web_png_path):
print(f"Copying {source_path} to {web_png_path}")
shutil.copy2(source_path, web_png_path)
else:
print(f"Skipping {source_path.name} (up to date)")
# Resolve metadata for this single PNG plot
plot_metadata = resolve_metadata_for_plot(source_path, {})
plot_metadata_cache[source_path.stem] = plot_metadata
items = [{
"name": source_path.stem,
"pdf_href": png_name, # Link to PNG instead of PDF
"png_href": png_name, # Thumbnail is the PNG
"metadata": plot_metadata
}]
# Save metadata cache for this directory
plot_cache = {source_path.stem: plot_metadata}
save_metadata_cache(source_web_dir, plot_cache)
# Calculate statistics for single file
current_stats = calculate_directory_stats(source_web_dir)
stats = {
"file_count": 1,
"folder_count": 0,
"total_size": format_file_size(current_stats["total_size"]),
"total_size_bytes": current_stats["total_size"]
}
# Calculate relative path to assets for single file
# Single files are at depth 1 (gallery_root/source.name/index.html)
assets_path = "../assets"
output_html = source_web_dir / "index.html"
with output_html.open("w") as f:
f.write(template.render(
title=source.name,
items=items,
subdirs=[],
relpath=source.name,
paths=CONFIG.paths,
ui=CONFIG.ui,
stats=stats,
folder_metadata={},
assets_path=assets_path
))
print(f"Generated {output_html}")
print(f"Processed {source.name}: {source.path}")
elif source_path.is_dir():
source_web_dir = gallery_root / source.name
source_web_dir.mkdir(parents=True, exist_ok=True)
@@ -544,20 +411,22 @@ def main(clean_first: bool = False) -> None:
print(f"Processed {source.name}: {source.path}")
else:
print(f"Warning: Source {source.path} is neither a "
f"directory nor a PDF/PNG file")
f"directory nor a PDF file")
print("Done")
# Generate root index.html for gallery navigation
generate_root_index(gallery_root)
# No copying of this script to CGI location
if __name__ == "__main__":
if 'GATEWAY_INTERFACE' in os.environ:
refresh_gallery_cgi()
else:
import argparse
parser = argparse.ArgumentParser(description='Generate gallery')
parser.add_argument('--clean', action='store_true',
help='Clean gallery directory before generation')
parser.add_argument(
'--clean',
action='store_true',
help='Clean gallery directory before generation'
)
args = parser.parse_args()
main(clean_first=args.clean)
pyproject.toml
+25
View File
@@ -0,0 +1,25 @@
[project]
name = "plot-gallery"
version = "0.1.0"
description = "Host your plots on a personal website"
authors = [
{ name = "K. Schmidt" }
]
readme = "README.md"
requires-python = ">=3.9"
[tool.black]
line-length = 120
target-version = ['py39']
[tool.isort]
profile = "black"
line_length = 120
[tool.flake8]
max-line-length = 120
extend-ignore = ["E203", "W503"]
[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"