From 5be91e0d17e16cfb01e17ebf4a8031c91e7d6428 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Thu, 25 Jun 2026 16:51:47 +0200 Subject: [PATCH] Expose dataset/conversion scripts as uv entry points scripts/ is now a proper package (scripts/__init__.py, added to the wheel's packages), with each script registered under [project.scripts] using its bare dashed name (e.g. `uv run migrate-geant-steps`). Tests now import these modules normally instead of loading them by file path. Co-Authored-By: Claude Sonnet 4.6 --- README.md | 12 ++++++++---- pyproject.toml | 7 ++++++- scripts/__init__.py | 0 tests/test_bump_dataset_version.py | 11 +---------- tests/test_create_root_files.py | 9 +-------- tests/test_steps_to_parquet.py | 11 +---------- tests/test_steps_to_parquet_parallel.py | 9 +-------- 7 files changed, 18 insertions(+), 41 deletions(-) create mode 100644 scripts/__init__.py diff --git a/README.md b/README.md index e0b3264..f30b903 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The model is developed in two phases: ## Data -Input: parquet files produced by [miniCaloSim](https://gitlab.etp.kit.edu/lbogner/minicalosim), or converted from a ROOT file via `scripts/steps_to_parquet.py`. Each row is one Geant4 step. Train/val split is by `event_id` (not row shuffle) to avoid leaking correlated steps from the same shower. +Input: parquet files produced by [miniCaloSim](https://gitlab.etp.kit.edu/lbogner/minicalosim), or converted from a ROOT file via `uv run steps-to-parquet`. Each row is one Geant4 step. Train/val split is by `event_id` (not row shuffle) to avoid leaking correlated steps from the same shower. ## Project structure @@ -55,9 +55,13 @@ giant/ │ ├── validate.py # step-level marginal + KL-divergence validation │ ├── analysis.py # notebook diagnostics: marginals, correlations, constraint checks │ └── cli.py # `giant train` / `giant predict` Typer app -├── scripts/ -│ ├── train.py # argparse training entry point -│ └── steps_to_parquet.py # ROOT → parquet conversion (uproot/awkward/polars) +├── scripts/ # also exposed as uv entry points, e.g. `uv run steps-to-parquet` +│ ├── steps_to_parquet.py # ROOT → parquet conversion (uproot/awkward/polars) +│ ├── steps_to_parquet_parallel.py # fan out steps_to_parquet.py over several ROOT files +│ ├── migrate_geant_steps.py # one-time move into the raw/processed/pools/derived layout +│ ├── bump_dataset_version.py # cut a new raw gen or parquet schema, with a logged reason +│ ├── create_root_files.py # generate new ROOT shards via a minicalosim executable +│ └── hparam_scan.py # hyperparameter grid scan over `giant train` runs └── tests/ ``` diff --git a/pyproject.toml b/pyproject.toml index 9d2fcf2..e6935e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,13 +37,18 @@ analysis = [ [project.scripts] giant = "giant.cli:app" +steps-to-parquet = "scripts.steps_to_parquet:main" +steps-to-parquet-parallel = "scripts.steps_to_parquet_parallel:main" +migrate-geant-steps = "scripts.migrate_geant_steps:main" +bump-dataset-version = "scripts.bump_dataset_version:main" +create-root-files = "scripts.create_root_files:main" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["giant"] +packages = ["giant", "scripts"] [tool.uv] conflicts = [ diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_bump_dataset_version.py b/tests/test_bump_dataset_version.py index e1de496..62c0f36 100644 --- a/tests/test_bump_dataset_version.py +++ b/tests/test_bump_dataset_version.py @@ -1,13 +1,4 @@ -import importlib.util -from pathlib import Path - -# scripts/ is not an installed package — load the module straight from its path. -_SPEC = importlib.util.spec_from_file_location( - "bump_dataset_version", - Path(__file__).resolve().parents[1] / "scripts" / "bump_dataset_version.py", -) -bump_dataset_version = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(bump_dataset_version) +from scripts import bump_dataset_version plan_bump_gen = bump_dataset_version.plan_bump_gen plan_bump_schema = bump_dataset_version.plan_bump_schema diff --git a/tests/test_create_root_files.py b/tests/test_create_root_files.py index 18d374f..ccd6047 100644 --- a/tests/test_create_root_files.py +++ b/tests/test_create_root_files.py @@ -1,17 +1,10 @@ -import importlib.util import json import stat from pathlib import Path import pytest -# scripts/ is not an installed package — load the module straight from its path. -_SPEC = importlib.util.spec_from_file_location( - "create_root_files", - Path(__file__).resolve().parents[1] / "scripts" / "create_root_files.py", -) -create_root_files = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(create_root_files) +from scripts import create_root_files parse_detector_spec = create_root_files.parse_detector_spec next_shard_index = create_root_files.next_shard_index diff --git a/tests/test_steps_to_parquet.py b/tests/test_steps_to_parquet.py index c095fd3..8f41a93 100644 --- a/tests/test_steps_to_parquet.py +++ b/tests/test_steps_to_parquet.py @@ -1,15 +1,6 @@ -import importlib.util -from pathlib import Path - import polars as pl -# scripts/ is not an installed package — load the module straight from its path. -_SPEC = importlib.util.spec_from_file_location( - "steps_to_parquet", - Path(__file__).resolve().parents[1] / "scripts" / "steps_to_parquet.py", -) -steps_to_parquet = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(steps_to_parquet) +from scripts import steps_to_parquet def _frame() -> pl.DataFrame: diff --git a/tests/test_steps_to_parquet_parallel.py b/tests/test_steps_to_parquet_parallel.py index 2daedf8..b5b2bb7 100644 --- a/tests/test_steps_to_parquet_parallel.py +++ b/tests/test_steps_to_parquet_parallel.py @@ -1,14 +1,7 @@ -import importlib.util import json from pathlib import Path -# scripts/ is not an installed package — load the module straight from its path. -_SPEC = importlib.util.spec_from_file_location( - "steps_to_parquet_parallel", - Path(__file__).resolve().parents[1] / "scripts" / "steps_to_parquet_parallel.py", -) -steps_to_parquet_parallel = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(steps_to_parquet_parallel) +from scripts import steps_to_parquet_parallel run_parallel = steps_to_parquet_parallel.run_parallel resolve_destination = steps_to_parquet_parallel.resolve_destination