From be99b3e601dd1cd8d5cbad60ff9e071fc2ad29b9 Mon Sep 17 00:00:00 2001 From: Kylian Schmidt Date: Mon, 8 Sep 2025 09:44:49 +0200 Subject: [PATCH] Add apptainer container. Clean up repo --- .gitignore | 5 +- .vscode/settings.json | 6 - Singularity.def | 20 +++ __pycache__/config.cpython-312.pyc | Bin 5718 -> 0 bytes orchestration/config.py | 2 - orchestration/pdf_export.py | 205 ------------------------- pyproject.toml | 4 + test.ipynb | 235 ----------------------------- 8 files changed, 28 insertions(+), 449 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 Singularity.def delete mode 100644 __pycache__/config.cpython-312.pyc delete mode 100644 orchestration/pdf_export.py delete mode 100644 test.ipynb diff --git a/.gitignore b/.gitignore index ed8ebf5..e6d7423 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -__pycache__ \ No newline at end of file +**__pycache__** +.vscode +*.sif +*.ipynb \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 47b059e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "flake8.args": [ - "--max-line-length=120", - "--ignore=W293,E123,W503", - ], -} \ No newline at end of file diff --git a/Singularity.def b/Singularity.def new file mode 100644 index 0000000..25dca20 --- /dev/null +++ b/Singularity.def @@ -0,0 +1,20 @@ +Bootstrap: docker +From: python:3.11-slim + +%post + apt-get update + apt-get install -y --no-install-recommends imagemagick + rm -rf /var/lib/apt/lists/* + + pip install --no-cache-dir jinja2 pyyaml + mkdir -p /src + +%files + . /src + +%environment + export PYTHONPATH=/src + +%runscript + cd /src + exec python3 generate_gallery.py "$@" \ No newline at end of file diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc deleted file mode 100644 index 9d7e9207ae3eb0fd9a86e3e13268665cfa31f846..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5718 zcmb_gU2Gf25#IaZ-;oq4>c_T}#HUF9q3hC4kTj`aq;>4dQDG}?oD_+G_IkQoQfD83 zx_7ipfocud0TduJdS|)W=Q+IX4)*T1xdc{=hu4P-U1+8M0sbzfKs+3ftyH}-JRNYysIvZav z1z&d>j!VmjRjXpybravL>F%s^SPd1Z>jK+3th_V{qo75~XvLtQfnUebs7|%cfKyQZ zgZAeoFA)yXixe$tn_g6ROv`Zdf+wGa{-+Go>=Y@e+#QF?5}{;TfPfKSf{>V&s7xcK zcnl=GCWSRQtVOWah~{ISf}Q&gojc__!gsvTV>$Bf)}5wvJ0ANF;AJJ zGg`4~n-uCs$l{aFVJGT=s zC0e0DF!KnpoSLmSR40g|f|sb-Raaxxs_VsSm6B%GEH6Ey8yD)e!0A)@7z0Eh4DDqZ zBv~ZANOC|t$*Q=l51ZWF0j%xn5Ue=JLNA?cWpY=hTIuYS)2+V2Tk@d)2I}nWp#30u zfP*j`xr1OuN4)q;llS*X-JuK?mP=VtH>f&8XZ4E~gdw`6Yb6!DY+1jgK_tSe*U&kR zmvgAjj9Ja3Gu1jQH?8bI!0qUiVHhuW%2t0MN--pR;qRdDe18n?S(o>;b`Rf@hkak5 z&KL4U7%bcu$sj}~0Z`_TzCgi52P^`ZQXN1L0I@E#;An-I6ogH|lX1a#vcI8o<6 z*%$D8fSwq$MB}d&70Mais%*{#Do-7d40Ko3OC?52x{Ha#wmTECu~2yz zoSf|>A6%t#Iy0fuGE9^;gSn@H$*D#ePeK^mW$MmWO+O*oRUNYm@5boPNotW*%jzq~ z&zx3^mW|m1N9;v#{v9tCgoP*8El3&}lNR;5?P{=)0InNeEXX-VkaWT0gkW8Q|J?$R zCDJ74;kmV(54IHD{BtBP%s=t$1C49&hy`4RCng4-MGAS5VYD&a3B3);&7pSD!EvaJ zV-oV`d;Li%Y-ANobPgidWzvq4Z2z0{ug%}ijICwHR^wwlOut0Kp87n7ot-*`k#dnT z2e%<7&rYIk2VjT{!S)Dv}a!14Vt!4JD#`k^MhJCK-!i89`Z4E^}UAR5i zHeJY^?+@(qUk-BZ&WQqf1rSmoumAFr^a`014B!~zLUNAeQ~GIeJqKU};nRTGThayx zYkdQ*3reVcoRysMu!Zl3;WDnC1vFRPDnEU2tOtzgI_+RgZS?d4kF718`7sOID96=d zBc#q*&?9CQYKEz*s|($NgVE;UOu$E`s_q1Aq|9$IR%PSrq<qY$FX8A#s-gLwRHCkbGT<9uzS|qY-f?e~PkWMKJM1y2dqUBZaqHzNJh^JTo(eLo z--^|(fE^@O-r6!MFX{`qB@Tu)&URRM#`6*wDjE(p#Luf4onT4C!0hZdFAuHmIi*DkFj zu0OuMtFV}Ae% z+Yyq;wh5@-QAoCLdEk2NooC;V-O4<-8h`HID2#eFJ@VgAc7p=nZ@)2kA|d^5_vncM z`S*#)iCk=I6L8{0ao}8p$`=PtH-yG!2oOzx&zsVX0C3@QMaEwTH1I-VD=ai*9~SZv zxTfgL={fx(8gSPFNWp=oW0fb&O~9ry7j7!;>YV1d4S6+#F+3jR zCH7Th<8oqr^M;;<{`{Daxm7>nI~$^)T>~Wt58M#%{hQ)Fm3?#awaK3qmtXv;y>k4$ zeIKO0$)o;udSop<@|lDA<&8(5mfnjT7h?H=9S?r|Rsc`K{>hgE0HVGp+W=xd+#n!$ z@nC=6;M-JxKgV}JxGR854ZKbA_w4=&`#APFfn>`Gdmr+UpfjA`0C|<%kt30KJ4I51 z>+(P=J+v+lwc>Et{%~afJMOKKW0y}YzWCNx*3yrx%g6o^N#7C0$iBN0iT1Ry){a9B zd1E1OEadGEdHX}&p^$e7dA)6MZ)8$f8Eq4MUO)H=f1*+#YCkI^!HFwzN00FOv$u@6hxj+_nwF!_wfXyB*kc5W- z=7fg=0X9>CAcKNTD98kYGlFnjXcPVjgg)yBB9T*z^M981w983~Vnl8L diff --git a/orchestration/config.py b/orchestration/config.py index 9672315..fc010cc 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -16,8 +16,6 @@ class PathConfig: """Configuration for system paths and directories.""" work_dir: str web_folder: str - cgi_script: str - config_path: str @dataclass diff --git a/orchestration/pdf_export.py b/orchestration/pdf_export.py deleted file mode 100644 index 51df501..0000000 --- a/orchestration/pdf_export.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -PDF Export Module for Scientific Gallery Generator - -This module handles exporting and merging multiple plots into a single PDF -using PyPDF2 and reportlab for layout. -""" - -import io -import tempfile -from pathlib import Path -from typing import List, Dict, Any -import subprocess - -try: - from PyPDF2 import PdfReader, PdfWriter - from reportlab.pdfgen import canvas - from reportlab.lib.pagesizes import letter, A4 - from reportlab.lib.utils import ImageReader - HAS_PDF_LIBS = True -except ImportError: - HAS_PDF_LIBS = False - - -class PDFExporter: - """Handles exporting multiple plots to a merged PDF""" - - def __init__(self): - self.page_size = A4 - self.margin = 50 - - def merge_plots(self, plot_paths: List[str], layout: Dict[str, int], - output_path: str = None) -> bytes: - """ - Merge multiple PDF plots into a single PDF with grid layout. - - Args: - plot_paths: List of paths to PDF files - layout: Dictionary with 'rows' and 'cols' keys - output_path: Optional output file path - - Returns: - PDF bytes - """ - if not HAS_PDF_LIBS: - return self._merge_with_pdfjam(plot_paths, layout, output_path) - - return self._merge_with_pypdf(plot_paths, layout, output_path) - - def _merge_with_pypdf(self, plot_paths: List[str], layout: Dict[str, int], - output_path: str = None) -> bytes: - """Merge PDFs using PyPDF2 and reportlab""" - from reportlab.pdfgen import canvas - from reportlab.lib.pagesizes import A4 - - # Create a new PDF with the layout - buffer = io.BytesIO() - c = canvas.Canvas(buffer, pagesize=A4) - page_width, page_height = A4 - - rows = layout['rows'] - cols = layout['cols'] - - # Calculate dimensions for each plot - plot_width = (page_width - 2 * self.margin) / cols - plot_height = (page_height - 2 * self.margin) / rows - - # Place each plot in the grid - for i, plot_path in enumerate(plot_paths): - if i >= rows * cols: - break - - row = i // cols - col = i % cols - - # Calculate position - x = self.margin + col * plot_width - y = page_height - self.margin - (row + 1) * plot_height - - try: - # Read the source PDF - with open(plot_path, 'rb') as f: - reader = PdfReader(f) - if len(reader.pages) > 0: - page = reader.pages[0] - - # Convert PDF page to image and place it - # This is a simplified approach - in practice you'd want - # to properly scale and position the PDF content - self._draw_pdf_placeholder(c, x, y, plot_width, plot_height, - Path(plot_path).stem) - - except Exception as e: - print(f"Error processing {plot_path}: {e}") - self._draw_error_placeholder(c, x, y, plot_width, plot_height) - - c.save() - - if output_path: - with open(output_path, 'wb') as f: - f.write(buffer.getvalue()) - - return buffer.getvalue() - - def _merge_with_pdfjam(self, plot_paths: List[str], layout: Dict[str, int], - output_path: str = None) -> bytes: - """Merge PDFs using pdfjam (requires pdfpages LaTeX package)""" - rows = layout['rows'] - cols = layout['cols'] - - # Create temporary output file - with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp_file: - temp_output = tmp_file.name - - try: - # Build pdfjam command - cmd = [ - 'pdfjam', - '--nup', f'{cols}x{rows}', - '--landscape' if cols > rows else '--no-landscape', - '--frame', 'true', - '--delta', '10pt 10pt', - '--offset', '0pt 0pt', - '--outfile', temp_output - ] - - # Add input files - cmd.extend(plot_paths[:rows * cols]) - - # Run pdfjam - result = subprocess.run(cmd, capture_output=True, text=True) - - if result.returncode != 0: - raise Exception(f"pdfjam failed: {result.stderr}") - - # Read the output file - with open(temp_output, 'rb') as f: - pdf_bytes = f.read() - - if output_path: - with open(output_path, 'wb') as f: - f.write(pdf_bytes) - - return pdf_bytes - - finally: - # Clean up temporary file - Path(temp_output).unlink(missing_ok=True) - - def _draw_pdf_placeholder(self, canvas, x: float, y: float, width: float, - height: float, plot_name: str): - """Draw a placeholder for a PDF plot""" - # Draw border - canvas.setStrokeColorRGB(0.5, 0.5, 0.5) - canvas.setLineWidth(1) - canvas.rect(x, y, width, height) - - # Draw plot name - canvas.setFillColorRGB(0, 0, 0) - canvas.setFont("Helvetica", 10) - text_width = canvas.stringWidth(plot_name, "Helvetica", 10) - text_x = x + (width - text_width) / 2 - text_y = y + height / 2 - canvas.drawString(text_x, text_y, plot_name) - - def _draw_error_placeholder(self, canvas, x: float, y: float, width: float, - height: float): - """Draw an error placeholder""" - # Draw red border - canvas.setStrokeColorRGB(1, 0, 0) - canvas.setLineWidth(2) - canvas.rect(x, y, width, height) - - # Draw error text - canvas.setFillColorRGB(1, 0, 0) - canvas.setFont("Helvetica-Bold", 12) - error_text = "Error loading plot" - text_width = canvas.stringWidth(error_text, "Helvetica-Bold", 12) - text_x = x + (width - text_width) / 2 - text_y = y + height / 2 - canvas.drawString(text_x, text_y, error_text) - - -def check_dependencies() -> Dict[str, bool]: - """Check if required dependencies are available""" - deps = { - 'pypdf2': HAS_PDF_LIBS, - 'pdfjam': False - } - - # Check for pdfjam - try: - result = subprocess.run(['pdfjam', '--version'], - capture_output=True, text=True) - deps['pdfjam'] = result.returncode == 0 - except FileNotFoundError: - pass - - return deps - - -if __name__ == "__main__": - # Test the exporter - exporter = PDFExporter() - deps = check_dependencies() - print("Available dependencies:", deps) diff --git a/pyproject.toml b/pyproject.toml index b519773..aa4edef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,10 @@ authors = [ ] readme = "README.md" requires-python = ">=3.9" +dependencies = [ + "jinja2", + "pyyaml", +] [tool.black] line-length = 120 diff --git a/test.ipynb b/test.ipynb deleted file mode 100644 index 10fe056..0000000 --- a/test.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "id": "aedbb9f5", - "metadata": {}, - "outputs": [], - "source": [ - "from dataclasses import dataclass, field, asdict\n", - "from pathlib import Path\n", - "\n", - "import yaml\n", - "\n", - "\n", - "@dataclass\n", - "class GalleryItem:\n", - " name: str\n", - " path: Path\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "66624faf", - "metadata": {}, - "outputs": [], - "source": [ - "yaml_file = Path(\"config.yaml\")\n", - "\n", - "with open(yaml_file, \"r\") as f:\n", - " new_config: dict = yaml.safe_load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "40a472f8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'test_1_plot',\n", - " 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/plot__proc_3_5606769559__cat_incl__var_jet1_pt.pdf'}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_config[\"sources\"][0]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "745c97ad", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[GalleryItem(name='test_1_plot', path=PosixPath('data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/plot__proc_3_5606769559__cat_incl__var_jet1_pt.pdf')),\n", - " GalleryItem(name='test_2_dir', path=PosixPath('data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1'))]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sources = [\n", - " GalleryItem(name=src[\"name\"], path=Path(src[\"path\"]))\n", - " for src in new_config.get(\"sources\", False)\n", - "]\n", - "sources" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "a0102f9d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Config(web_folder='', backup_folder='', png_dpi=400, plot_root='gallery', sources=[{'name': 'test_1_plot', 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/plot__proc_3_5606769559__cat_incl__var_jet1_pt.pdf'}, {'name': 'test_2_dir', 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/'}])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\n", - "\n", - "@dataclass\n", - "class Config:\n", - " web_folder: str = \"\"\n", - " backup_folder: str = \"\"\n", - " png_dpi: int = 400\n", - " plot_root: str = \"gallery\"\n", - " sources: list[GalleryItem] = field(default_factory=list)\n", - "\n", - " @classmethod\n", - " def from_yaml(cls, yaml_file: str, strict: bool = False) -> \"Config\":\n", - " \"\"\"\n", - " Load configuration from a YAML file.\n", - "\n", - " Args:\n", - " yaml_file (str): Path to the YAML file.\n", - " strict (bool):\n", - " If True, raises an error if a key in the YAML file does not exist in the Config class.\n", - " If False (default), adds all keys as attributes\n", - "\n", - " Returns:\n", - " Config: Instance of this class\n", - " \"\"\"\n", - "\n", - " with open(yaml_file, \"r\") as f:\n", - " new_config: dict = yaml.safe_load(f)\n", - "\n", - " new_config[\"sources\"] = [\n", - " GalleryItem(name=src[\"name\"], path=Path(src[\"path\"]))\n", - " for src in new_config.get(\"sources\", False)\n", - " ]\n", - "\n", - " instance = cls(**{\n", - " k: v\n", - " for k, v in new_config.items()\n", - " if hasattr(cls, k) or not strict\n", - " })\n", - "\n", - " for key in new_config.keys():\n", - " if strict and not hasattr(instance, key):\n", - " raise KeyError(f\"Key '{key}' not found in Config class\")\n", - "\n", - " return instance\n", - "\n", - " def to_yaml(self, yaml_file: str) -> None:\n", - " \"\"\"\n", - " Save the current configuration to a YAML file.\n", - "\n", - " Args:\n", - " yaml_file (str): Path to the YAML file.\n", - " \"\"\"\n", - " with open(yaml_file, \"w\") as f:\n", - " yaml.dump(asdict(self), f, default_flow_style=False)\n", - "\n", - "\n", - "instance = Config(**{\n", - " k: v\n", - " for k, v in new_config.items()\n", - " #if hasattr(Config, k)\n", - "})\n", - "instance" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "0b516ae9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'test_1_plot',\n", - " 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/plot__proc_3_5606769559__cat_incl__var_jet1_pt.pdf'},\n", - " {'name': 'test_2_dir',\n", - " 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/'}]" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "new_config[\"sources\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "b047f0c5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'test_1_plot',\n", - " 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/plot__proc_3_5606769559__cat_incl__var_jet1_pt.pdf'},\n", - " {'name': 'test_2_dir',\n", - " 'path': 'data/cf_store/test_analysis/cf.PlotVariables1D/run2_2016_nano_v9/calib__test/sel__test/prod__test/weight__test/nominal/datasets_ttbar_dl/dev1/'}]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "instance.sources" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.1" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}