Fix CLI/tooling robustness gaps and dedupe the Conditioning enum
CI / Lint (ruff check) (push) Successful in 35s
CI / Format (ruff format) (push) Failing after 31s
CI / Type check (ty) (push) Successful in 30s
CI / Sync project version with tag (push) Has been skipped
CI / Tests (push) Successful in 1m7s

- run_create_manifest gains --force; it previously overwrote an
  existing manifest (including holdout.manifest, which
  check_holdout_overlap exists specifically to protect) with no
  warning or backup on a second run.
- _git_user_name only caught OSError, not subprocess.TimeoutExpired (a
  SubprocessError, not an OSError) — a slow/loaded shared portal
  machine could crash `dwarf bump-gen`/`bump-schema` instead of
  degrading to by=None as intended.
- `dwarf convert --jobs`/`make-root --jobs` now warn (never block) when
  the requested count exceeds ~1/4 of the machine's CPUs, matching the
  same shared-machine etiquette check added to giant train in the
  previous commit.
- The Conditioning enum was independently redefined in both
  giant/cli.py and scripts/dwarf.py; moved to a single
  giant.config.Conditioning both now import, removing the drift risk
  of a third conditioning mode being added to one but not the other.

Each fix has a regression test.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-03 13:49:09 +02:00
parent ad1b8e7835
commit ca3a2a3462
6 changed files with 124 additions and 10 deletions
+28 -5
View File
@@ -5,6 +5,7 @@ simulation-fanout tools into one Typer app so there's a single command name
(and `--help`) to remember instead of five differently-hyphenated ones.
"""
import os
from enum import Enum
from pathlib import Path
from typing import Optional
@@ -12,6 +13,7 @@ from typing import Optional
import typer
from typing_extensions import Annotated
from giant.config import Conditioning
from scripts.bump_dataset_version import (
run_bump_gen,
run_bump_schema,
@@ -37,6 +39,25 @@ def _main() -> None:
"""dwarf — dataset/tooling CLI (ROOT<->parquet conversion, dataset versioning, sim fanout)."""
def _warn_if_exceeds_shared_quota(n: int, flag: str) -> None:
"""Soft warning (never blocks) when a worker/job count looks likely to
grab more than this repo's documented shared-portal-machine etiquette
(CLAUDE.md's Compute environment: stay within ~1/4 of CPU/RAM and a
single GPU, since portal1/deepthought{,2}/bms{1..3} are shared with
other users). Not a hard cap — a legitimate big machine or a
deliberately aggressive run is still the caller's call.
"""
cpu_count = os.cpu_count() or 1
quota = max(1, cpu_count // 4)
if n > quota:
typer.echo(
f"warning: {flag}={n} exceeds ~1/4 of this machine's "
f"{cpu_count} CPU(s) ({quota}) — portal machines are shared "
"with other users (see CLAUDE.md's Compute environment section)",
err=True,
)
class Compression(str, Enum):
snappy = "snappy"
lz4 = "lz4"
@@ -106,6 +127,7 @@ def convert(
if jobs < 1:
typer.echo("error: --jobs must be >= 1", err=True)
raise typer.Exit(1)
_warn_if_exceeds_shared_quota(jobs, "--jobs")
compression_value = (
"uncompressed" if compression is Compression.none else compression.value
@@ -327,6 +349,10 @@ def create_manifest(
execute: Annotated[
bool, typer.Option("--execute", help="Write the manifest (default: dry run)")
] = False,
force: Annotated[
bool,
typer.Option("--force", help="Overwrite the manifest if it already exists"),
] = False,
) -> None:
"""Create a new manifest from a list of parquet files."""
run_create_manifest(
@@ -336,6 +362,7 @@ def create_manifest(
pool=pool,
type_=type_.value if type_ is not None else None,
root=str(root),
force=force,
)
@@ -391,6 +418,7 @@ def make_root(
] = False,
) -> None:
"""Generate new ROOT shards via a minicalosim executable."""
_warn_if_exceeds_shared_quota(jobs, "--jobs")
run_make_root(
executable=executable,
detector=detector,
@@ -472,11 +500,6 @@ def build_geometry_oracle(
)
class Conditioning(str, Enum):
physical = "physical"
embedding = "embedding"
@app.command("warm-cache")
def warm_cache(
data: Annotated[