From f427d3384fdb936c3215c9cf8cabd0fe605aa1de Mon Sep 17 00:00:00 2001
From: Lars Bogner <upyxo@student.kit.edu>
Date: Wed, 29 Jul 2026 10:42:40 +0200
Subject: [PATCH] Timestamp default checkpoint dir to avoid W&B run-id
 collisions

out_dir (and thus the W&B run id, which is derived from out_dir.name)
was previously date-only, so two fresh runs on the same day with
identical hyperparams silently shared one W&B run history. Default
out_dir is now timestamped to the second. --resume without an explicit
--out now reuses the checkpoint's own parent directory instead of
recomputing a hyperparam-derived name, which both preserves the old
continue-in-place behavior and fixes a latent bug where a resumed run
with a changed hyperparam (e.g. --lr) would silently start writing to
a new directory.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 giant/cli.py | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/giant/cli.py b/giant/cli.py
index 2e1d6c2..327ec5e 100644
--- a/giant/cli.py
+++ b/giant/cli.py
@@ -1,5 +1,5 @@
 from collections import Counter
-from datetime import date, datetime, timezone
+from datetime import datetime, timezone
 from enum import Enum
 import math
 from pathlib import Path
@@ -388,7 +388,10 @@ def train(
     out: Annotated[
         Optional[Path],
         typer.Option(
-            "--out", "-o", help="Checkpoint dir (default: auto from hyperparams)"
+            "--out",
+            "-o",
+            help="Checkpoint dir (default: timestamped dir from hyperparams, "
+            "or the --resume checkpoint's own dir when resuming)",
         ),
     ] = None,
     device: Annotated[
@@ -513,16 +516,28 @@ def train(
             f"batch_size: {t['batch_size']} (auto-estimated from free GPU memory)"
         )
 
-    out_dir = out or Path(
-        f"checkpoints/{date.today().strftime('%Y%m%d')}"
-        f"_{t['mode']}"
-        f"_h{m['hidden_dim']}"
-        f"_b{m['n_blocks']}"
-        f"_e{m['emb_dim']}"
-        f"_c{m['conditioning']}"
-        f"_lr{t['lr']}"
-        f"_bs{t['batch_size']}"
-    )
+    if out is not None:
+        out_dir = out
+    elif resume is not None:
+        # Continue writing into the resumed checkpoint's own directory
+        # rather than recomputing a hyperparam-derived name — the latter
+        # would (a) collide with the original run's dir only by accident
+        # (same day, unchanged hyperparams) and now never collides at all
+        # since the fresh-run name below is timestamped to the second, and
+        # (b) silently start a fresh directory if a resumed run tweaks any
+        # hyperparam baked into the name (e.g. --lr for a fine-tune).
+        out_dir = resume.parent
+    else:
+        out_dir = Path(
+            f"checkpoints/{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            f"_{t['mode']}"
+            f"_h{m['hidden_dim']}"
+            f"_b{m['n_blocks']}"
+            f"_e{m['emb_dim']}"
+            f"_c{m['conditioning']}"
+            f"_lr{t['lr']}"
+            f"_bs{t['batch_size']}"
+        )
 
     typer.echo(f"device:  {_device}")
     typer.echo(f"out_dir: {out_dir}")