Fix training-loop checkpoint/resume and WGAN bugs

- Graceful shutdown (SIGINT/SIGTERM) now actually saves a checkpoint of
  in-progress weights before exiting mid-epoch — it previously broke
  out of the epoch loop before reaching the checkpoint-save block,
  contradicting its own printed "saving a checkpoint" message and
  losing all progress since the last completed epoch. Checkpoint-dict
  construction is factored into a shared _build_checkpoint() helper
  used by both the mid-epoch and end-of-epoch save paths.
- WGAN LR-schedule steps_per_epoch used the wrong denominator
  (n_critic + 1 instead of n_critic), causing the schedule to exhaust
  early and LR to floor to 0 before training completed.
- --critic-lr override was silently dropped on WGAN --resume (only the
  generator optimizer's LR was made authoritative again after
  load_state_dict; optimizer_d's was not).
- WGAN secondary gradient-penalty forced x_hat/grad to zero for
  fully-masked rows (n_sec == 0, common in a shower), adding a
  constant ~1.0 bias into the batch-mean GP term; such rows are now
  excluded from the mean.
- run_train_job warns (never blocks) when --num-workers exceeds ~1/4
  of the machine's CPUs, per this repo's shared-portal-machine
  etiquette (see CLAUDE.md's Compute environment section).

Each fix has a regression test.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

tests/test_pipeline.py

@@ -108,13 +108,13 @@ def _tiny_cfg(**train_overrides):
 
 def _run(data, out_dir, cfg=None, **kwargs):
     echoed: list[str] = []
+    kwargs.setdefault("num_workers", 0)
     run_train_job(
         data=data,
         cfg=cfg or _tiny_cfg(),
         out_dir=out_dir,
         device=torch.device("cpu"),
         shuffle_buffer=64,
-        num_workers=0,
         echo=echoed.append,
         **kwargs,
     )
@@ -143,6 +143,22 @@ def test_run_train_job_second_run_hits_cache(tmp_path, data, monkeypatch):
     assert "normalizer: cache hit" in joined
 
 
+def test_run_train_job_warns_when_num_workers_exceeds_shared_quota(
+    tmp_path, data, monkeypatch
+):
+    monkeypatch.setattr("giant.pipeline.os.cpu_count", lambda: 8)  # quota = 2
+    echo = _run(data, tmp_path / "out", num_workers=3)
+    assert any("num-workers=3" in m and "exceeds" in m for m in echo)
+
+
+def test_run_train_job_no_warning_when_num_workers_within_shared_quota(
+    tmp_path, data, monkeypatch
+):
+    monkeypatch.setattr("giant.pipeline.os.cpu_count", lambda: 8)  # quota = 2
+    echo = _run(data, tmp_path / "out", num_workers=2)
+    assert not any("exceeds" in m for m in echo)
+
+
 def test_run_train_job_no_cache_setup_never_writes_sidecar(tmp_path, data):
     _run(data, tmp_path / "out", cache_setup=False)
     assert not setup_cache.sidecar_path(data).exists()