From 61410ddee37831c4e83be2fc42ba8b2d1011fa7a Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Mon, 27 Jul 2026 11:29:35 +0200 Subject: [PATCH] analyze: default run directory to /analysis_runs, gitignored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit giant analyze prep/submit previously defaulted the run directory to next to the rollout parquet on /ceph. Default it instead to /analysis_runs/analysis_ so it lands inside the portal repo checkout (/work) — gitignored, --run-dir still overrides it. derive_run_dir/prep gained a default_base param; library callers that don't pass one keep the old parquet-relative fallback. --- .gitignore | 3 +++ giant/analysis/condor.py | 25 ++++++++++++++++++++----- giant/cli.py | 10 ++++++++-- tests/test_condor.py | 9 +++++++++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 377b0b7..95aeaf8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ checkpoints/ # Scratch working directory /scratchpad/ + +# giant analyze run directories (shared.json, reduced/, plots/, condor logs) +/analysis_runs/ diff --git a/giant/analysis/condor.py b/giant/analysis/condor.py index 13c5183..039c6e9 100644 --- a/giant/analysis/condor.py +++ b/giant/analysis/condor.py @@ -93,13 +93,26 @@ def load_rollout_yaml(path: str | Path) -> dict: return d -def derive_run_dir(rollout_yaml: dict, run_dir: str | Path | None = None) -> Path: - """Analysis output directory, next to the rollout parquet unless overridden.""" +def derive_run_dir( + rollout_yaml: dict, + run_dir: str | Path | None = None, + default_base: str | Path | None = None, +) -> Path: + """Analysis output directory. + + Precedence: an explicit ``run_dir`` always wins. Otherwise + ``default_base / analysis_`` if ``default_base`` is given (the CLI + passes the repo's gitignored ``analysis_runs/``, so run directories don't + pile up on ``/ceph`` next to the rollout parquet). Falls back to next to + the rollout parquet — the original convention — for callers that don't + care where the run directory lives. + """ if run_dir is not None: return Path(run_dir) rollout = Path(rollout_yaml["output"]) tag = str(rollout_yaml.get("prediction_id") or rollout.stem)[:8] - return rollout.parent / f"analysis_{tag}" + base = Path(default_base) if default_base is not None else rollout.parent + return base / f"analysis_{tag}" def _plot_meta(rollout_yaml: dict) -> dict: @@ -160,6 +173,7 @@ def prep( rollout_yaml: str | Path, run_dir: str | Path | None = None, n_chunks: int = 1, + default_base: str | Path | None = None, **ctx_kwargs, ) -> Path: """Read the rollout YAML, build the shared context, and lay out the run dir. @@ -168,10 +182,11 @@ def prep( ``n_chunks`` is the run-level chunk count every ``compute-one``/``merge-one`` job reads back out of ``run_meta.json`` (via ``RunMeta.n_chunks``), so it is resolved once here rather than re-passed (and risking disagreement) at every - later step. + later step. See ``derive_run_dir`` for how ``run_dir``/``default_base`` + resolve the actual directory. """ y = load_rollout_yaml(rollout_yaml) - run_path = derive_run_dir(y, run_dir) + run_path = derive_run_dir(y, run_dir, default_base=default_base) run_path.mkdir(parents=True, exist_ok=True) rollout, reference = y["output"], y["dataset"] diff --git a/giant/cli.py b/giant/cli.py index 9958157..511bad2 100644 --- a/giant/cli.py +++ b/giant/cli.py @@ -1144,7 +1144,7 @@ def analyze_prep( typer.Option( "--run-dir", "-o", - help="Override the run directory (default: next to the rollout parquet)", + help="Override the run directory (default: /analysis_runs/analysis_)", ), ] = None, n_energy_bins: Annotated[int, typer.Option("--energy-bins")] = 4, @@ -1164,6 +1164,7 @@ def analyze_prep( rollout_yaml, run_dir, n_chunks=chunks, + default_base=Path.cwd() / "analysis_runs", n_energy_bins=n_energy_bins, n_marginal_bins=n_marginal_bins, top_k_pdg=top_k_pdg, @@ -1244,7 +1245,11 @@ def analyze_submit( accounting_group: Annotated[str, typer.Option("--accounting-group")], run_dir: Annotated[ Optional[Path], - typer.Option("--run-dir", "-o", help="Override the run directory"), + typer.Option( + "--run-dir", + "-o", + help="Override the run directory (default: /analysis_runs/analysis_)", + ), ] = None, docker_image: Annotated[ str, typer.Option("--docker-image") @@ -1277,6 +1282,7 @@ def analyze_submit( rollout_yaml, run_dir, n_chunks=chunks, + default_base=Path.cwd() / "analysis_runs", n_energy_bins=n_energy_bins, n_marginal_bins=n_marginal_bins, top_k_pdg=top_k_pdg, diff --git a/tests/test_condor.py b/tests/test_condor.py index fd175b6..c4259ac 100644 --- a/tests/test_condor.py +++ b/tests/test_condor.py @@ -89,6 +89,15 @@ def test_derive_run_dir_next_to_rollout(): assert derive_run_dir(y, "/somewhere") == Path("/somewhere") +def test_derive_run_dir_default_base(): + y = {"output": "/data/roll.parquet", "prediction_id": "abcd1234ef", "dataset": "d"} + assert derive_run_dir(y, default_base="/work/lbogner/giant2/analysis_runs") == Path( + "/work/lbogner/giant2/analysis_runs/analysis_abcd1234" + ) + # an explicit run_dir still wins over default_base + assert derive_run_dir(y, "/somewhere", default_base="/other") == Path("/somewhere") + + def test_prep_lays_out_run_dir(tmp_path: Path): yaml_path = _write_inputs(tmp_path) run_dir = _prep(yaml_path)