Fix CLI/tooling robustness gaps and dedupe the Conditioning enum
- run_create_manifest gains --force; it previously overwrote an existing manifest (including holdout.manifest, which check_holdout_overlap exists specifically to protect) with no warning or backup on a second run. - _git_user_name only caught OSError, not subprocess.TimeoutExpired (a SubprocessError, not an OSError) — a slow/loaded shared portal machine could crash `dwarf bump-gen`/`bump-schema` instead of degrading to by=None as intended. - `dwarf convert --jobs`/`make-root --jobs` now warn (never block) when the requested count exceeds ~1/4 of the machine's CPUs, matching the same shared-machine etiquette check added to giant train in the previous commit. - The Conditioning enum was independently redefined in both giant/cli.py and scripts/dwarf.py; moved to a single giant.config.Conditioning both now import, removing the drift risk of a third conditioning mode being added to one but not the other. Each fix has a regression test. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from scripts import bump_dataset_version
|
||||
|
||||
plan_bump_gen = bump_dataset_version.plan_bump_gen
|
||||
@@ -11,6 +13,14 @@ apply_create_manifest = bump_dataset_version.apply_create_manifest
|
||||
check_holdout_overlap = bump_dataset_version.check_holdout_overlap
|
||||
|
||||
|
||||
def test_git_user_name_returns_none_on_timeout(monkeypatch):
|
||||
def _raise_timeout(*args, **kwargs):
|
||||
raise subprocess.TimeoutExpired(cmd=["git"], timeout=2)
|
||||
|
||||
monkeypatch.setattr(subprocess, "run", _raise_timeout)
|
||||
assert bump_dataset_version._git_user_name() is None
|
||||
|
||||
|
||||
def test_bump_gen_starts_at_gen1_when_none_exist(tmp_path):
|
||||
dirs, log_line = plan_bump_gen(
|
||||
tmp_path, "steps", "first generation", None, "2026-01-01"
|
||||
@@ -401,6 +411,36 @@ def test_create_manifest_creates_parent_dirs(tmp_path):
|
||||
assert output.exists()
|
||||
|
||||
|
||||
def test_run_create_manifest_refuses_to_overwrite_existing_output(tmp_path):
|
||||
pq = tmp_path / "a.parquet"
|
||||
pq.touch()
|
||||
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
|
||||
output.parent.mkdir(parents=True)
|
||||
output.write_text("original contents\n")
|
||||
|
||||
try:
|
||||
bump_dataset_version.run_create_manifest(
|
||||
[str(pq)], execute=True, output=str(output)
|
||||
)
|
||||
assert False, "expected SystemExit"
|
||||
except SystemExit:
|
||||
pass
|
||||
assert output.read_text() == "original contents\n"
|
||||
|
||||
|
||||
def test_run_create_manifest_force_overwrites_existing_output(tmp_path):
|
||||
pq = tmp_path / "a.parquet"
|
||||
pq.touch()
|
||||
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
|
||||
output.parent.mkdir(parents=True)
|
||||
output.write_text("original contents\n")
|
||||
|
||||
bump_dataset_version.run_create_manifest(
|
||||
[str(pq)], execute=True, output=str(output), force=True
|
||||
)
|
||||
assert output.read_text() != "original contents\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_holdout_overlap
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,12 +1,35 @@
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from giant import cli as giant_cli
|
||||
from giant.config import Conditioning
|
||||
from giant.data import setup_cache
|
||||
from scripts import dwarf
|
||||
from scripts.dwarf import app
|
||||
from test_pipeline import _make_synthetic_steps
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_conditioning_enum_shared_across_both_clis():
|
||||
"""giant.cli and scripts.dwarf must use the one giant.config.Conditioning
|
||||
enum, not independently redefined copies that could silently drift apart
|
||||
on valid --conditioning values."""
|
||||
assert dwarf.Conditioning is Conditioning
|
||||
assert giant_cli.Conditioning is Conditioning
|
||||
|
||||
|
||||
def test_warn_if_exceeds_shared_quota_warns_over_quarter_cpu(monkeypatch, capsys):
|
||||
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
|
||||
dwarf._warn_if_exceeds_shared_quota(3, "--jobs")
|
||||
assert "warning: --jobs=3 exceeds" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_warn_if_exceeds_shared_quota_silent_within_quota(monkeypatch, capsys):
|
||||
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
|
||||
dwarf._warn_if_exceeds_shared_quota(2, "--jobs")
|
||||
assert capsys.readouterr().err == ""
|
||||
|
||||
|
||||
def test_convert_rejects_jobs_below_one(tmp_path):
|
||||
root_file = tmp_path / "shard.root"
|
||||
root_file.touch()
|
||||
|
||||
Reference in New Issue
Block a user