Persist global_step across --resume so W&B step stays monotonic
Previously global_step always reset to 0, even on --resume. Since the W&B run reattaches to the same run id on resume, logging with step=global_step after a restart passed step values below what was already recorded, silently dropping the resumed portion's metrics. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+8
-1
@@ -483,6 +483,7 @@ def train(
|
||||
|
||||
start_epoch = 1
|
||||
best_val_loss = float("inf")
|
||||
resumed_global_step = 0
|
||||
if resume_path is not None:
|
||||
ckpt = torch.load(resume_path, map_location=device, weights_only=False)
|
||||
stage1_model.load_state_dict(ckpt["model"])
|
||||
@@ -506,6 +507,7 @@ def train(
|
||||
lr_sched.load_state_dict(ckpt["lr_sched"])
|
||||
start_epoch = ckpt.get("epoch", 0) + 1
|
||||
best_val_loss = ckpt.get("best_val_loss", float("inf"))
|
||||
resumed_global_step = ckpt.get("global_step", 0)
|
||||
|
||||
# optimizer/lr_sched.load_state_dict() above restore the checkpoint's
|
||||
# own base LR, which would otherwise silently override an explicit
|
||||
@@ -535,7 +537,11 @@ def train(
|
||||
|
||||
epoch_w = len(str(epochs))
|
||||
last_completed_epoch = start_epoch - 1
|
||||
global_step = 0
|
||||
# Restored from the checkpoint on --resume so wandb_run.log(..., step=...)
|
||||
# keeps advancing monotonically instead of restarting at 0 mid-run (a
|
||||
# reattached wandb run — see wandb.init(id=..., resume="allow") below —
|
||||
# would otherwise silently drop every post-resume point).
|
||||
global_step = resumed_global_step
|
||||
with _GracefulShutdown() as shutdown:
|
||||
for epoch in range(start_epoch, epochs + 1):
|
||||
epoch_start = time.monotonic()
|
||||
@@ -950,6 +956,7 @@ def train(
|
||||
"lr_sched": lr_sched.state_dict(),
|
||||
"epoch": epoch,
|
||||
"best_val_loss": best_val_loss,
|
||||
"global_step": global_step,
|
||||
}
|
||||
if mode == "wgan":
|
||||
assert (
|
||||
|
||||
Reference in New Issue
Block a user