Add configurable dropout to ResBlocks
Wire a dropout hyperparameter (default 0.1) through the config, model, training pipeline, and CLI. Persisted in saved model_config so checkpoints reconstruct the architecture correctly. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -49,18 +49,20 @@ class ConditionEncoder(nn.Module):
|
||||
|
||||
|
||||
class ResBlock(nn.Module):
|
||||
def __init__(self, dim: int, cond_dim: int) -> None:
|
||||
def __init__(self, dim: int, cond_dim: int, dropout: float = 0.1) -> None:
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim)
|
||||
self.linear1 = nn.Linear(dim, dim)
|
||||
self.cond_proj = nn.Linear(cond_dim, dim, bias=False)
|
||||
self.act = nn.SiLU()
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.linear2 = nn.Linear(dim, dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
||||
h = self.norm(x)
|
||||
h = self.linear1(h) + self.cond_proj(cond)
|
||||
h = self.act(h)
|
||||
h = self.dropout(h)
|
||||
h = self.linear2(h)
|
||||
return x + h
|
||||
|
||||
@@ -76,6 +78,7 @@ class DenoisingMLP(nn.Module):
|
||||
time_dim: int = 64,
|
||||
cond_out_dim: int = 128,
|
||||
x_dim: int = X_DIM,
|
||||
dropout: float = 0.1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.time_emb = SinusoidalEmbedding(time_dim)
|
||||
@@ -88,7 +91,7 @@ class DenoisingMLP(nn.Module):
|
||||
merged_cond_dim = time_dim + cond_out_dim
|
||||
self.input_proj = nn.Linear(x_dim, hidden_dim)
|
||||
self.blocks = nn.ModuleList([
|
||||
ResBlock(hidden_dim, merged_cond_dim) for _ in range(n_blocks)
|
||||
ResBlock(hidden_dim, merged_cond_dim, dropout=dropout) for _ in range(n_blocks)
|
||||
])
|
||||
self.out_proj = nn.Linear(hidden_dim, x_dim)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user