Timestamp default checkpoint dir to avoid W&B run-id collisions
out_dir (and thus the W&B run id, which is derived from out_dir.name) was previously date-only, so two fresh runs on the same day with identical hyperparams silently shared one W&B run history. Default out_dir is now timestamped to the second. --resume without an explicit --out now reuses the checkpoint's own parent directory instead of recomputing a hyperparam-derived name, which both preserves the old continue-in-place behavior and fixes a latent bug where a resumed run with a changed hyperparam (e.g. --lr) would silently start writing to a new directory. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+27
-12
@@ -1,5 +1,5 @@
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
import math
|
||||
from pathlib import Path
|
||||
@@ -388,7 +388,10 @@ def train(
|
||||
out: Annotated[
|
||||
Optional[Path],
|
||||
typer.Option(
|
||||
"--out", "-o", help="Checkpoint dir (default: auto from hyperparams)"
|
||||
"--out",
|
||||
"-o",
|
||||
help="Checkpoint dir (default: timestamped dir from hyperparams, "
|
||||
"or the --resume checkpoint's own dir when resuming)",
|
||||
),
|
||||
] = None,
|
||||
device: Annotated[
|
||||
@@ -513,16 +516,28 @@ def train(
|
||||
f"batch_size: {t['batch_size']} (auto-estimated from free GPU memory)"
|
||||
)
|
||||
|
||||
out_dir = out or Path(
|
||||
f"checkpoints/{date.today().strftime('%Y%m%d')}"
|
||||
f"_{t['mode']}"
|
||||
f"_h{m['hidden_dim']}"
|
||||
f"_b{m['n_blocks']}"
|
||||
f"_e{m['emb_dim']}"
|
||||
f"_c{m['conditioning']}"
|
||||
f"_lr{t['lr']}"
|
||||
f"_bs{t['batch_size']}"
|
||||
)
|
||||
if out is not None:
|
||||
out_dir = out
|
||||
elif resume is not None:
|
||||
# Continue writing into the resumed checkpoint's own directory
|
||||
# rather than recomputing a hyperparam-derived name — the latter
|
||||
# would (a) collide with the original run's dir only by accident
|
||||
# (same day, unchanged hyperparams) and now never collides at all
|
||||
# since the fresh-run name below is timestamped to the second, and
|
||||
# (b) silently start a fresh directory if a resumed run tweaks any
|
||||
# hyperparam baked into the name (e.g. --lr for a fine-tune).
|
||||
out_dir = resume.parent
|
||||
else:
|
||||
out_dir = Path(
|
||||
f"checkpoints/{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||||
f"_{t['mode']}"
|
||||
f"_h{m['hidden_dim']}"
|
||||
f"_b{m['n_blocks']}"
|
||||
f"_e{m['emb_dim']}"
|
||||
f"_c{m['conditioning']}"
|
||||
f"_lr{t['lr']}"
|
||||
f"_bs{t['batch_size']}"
|
||||
)
|
||||
|
||||
typer.echo(f"device: {_device}")
|
||||
typer.echo(f"out_dir: {out_dir}")
|
||||
|
||||
Reference in New Issue
Block a user