Timestamp default checkpoint dir to avoid W&B run-id collisions

out_dir (and thus the W&B run id, which is derived from out_dir.name)
was previously date-only, so two fresh runs on the same day with
identical hyperparams silently shared one W&B run history. Default
out_dir is now timestamped to the second. --resume without an explicit
--out now reuses the checkpoint's own parent directory instead of
recomputing a hyperparam-derived name, which both preserves the old
continue-in-place behavior and fixes a latent bug where a resumed run
with a changed hyperparam (e.g. --lr) would silently start writing to
a new directory.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-29 10:42:40 +02:00
parent bb699d41b2
commit f427d3384f
+27 -12
View File
@@ -1,5 +1,5 @@
from collections import Counter
from datetime import date, datetime, timezone
from datetime import datetime, timezone
from enum import Enum
import math
from pathlib import Path
@@ -388,7 +388,10 @@ def train(
out: Annotated[
Optional[Path],
typer.Option(
"--out", "-o", help="Checkpoint dir (default: auto from hyperparams)"
"--out",
"-o",
help="Checkpoint dir (default: timestamped dir from hyperparams, "
"or the --resume checkpoint's own dir when resuming)",
),
] = None,
device: Annotated[
@@ -513,16 +516,28 @@ def train(
f"batch_size: {t['batch_size']} (auto-estimated from free GPU memory)"
)
out_dir = out or Path(
f"checkpoints/{date.today().strftime('%Y%m%d')}"
f"_{t['mode']}"
f"_h{m['hidden_dim']}"
f"_b{m['n_blocks']}"
f"_e{m['emb_dim']}"
f"_c{m['conditioning']}"
f"_lr{t['lr']}"
f"_bs{t['batch_size']}"
)
if out is not None:
out_dir = out
elif resume is not None:
# Continue writing into the resumed checkpoint's own directory
# rather than recomputing a hyperparam-derived name — the latter
# would (a) collide with the original run's dir only by accident
# (same day, unchanged hyperparams) and now never collides at all
# since the fresh-run name below is timestamped to the second, and
# (b) silently start a fresh directory if a resumed run tweaks any
# hyperparam baked into the name (e.g. --lr for a fine-tune).
out_dir = resume.parent
else:
out_dir = Path(
f"checkpoints/{datetime.now().strftime('%Y%m%d_%H%M%S')}"
f"_{t['mode']}"
f"_h{m['hidden_dim']}"
f"_b{m['n_blocks']}"
f"_e{m['emb_dim']}"
f"_c{m['conditioning']}"
f"_lr{t['lr']}"
f"_bs{t['batch_size']}"
)
typer.echo(f"device: {_device}")
typer.echo(f"out_dir: {out_dir}")