tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2026-05-11 18:30:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #686 from marcinbogdanski/fix/init-smear-backout-lambdas

Browse Source 
Initialize smear and backout lambdas in init_weights()
This commit is contained in:
Andrej 2026-04-13 16:08:04 -07:00 committed by GitHub
commit b9b6ce137b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nanochat/gpt.py
View File

@ -238,6 +238,11 @@ class GPT(nn.Module):
for i in range(n_layer):
self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
# Smear/backout scalars and smear gate must be explicitly initialized
torch.nn.init.zeros_(self.smear_lambda)
torch.nn.init.constant_(self.backout_lambda, 0.2)
torch.nn.init.uniform_(self.smear_gate.weight, 0.0, 0.02)
# Value embeddings (init like c_v: uniform with same std)
for ve in self.value_embeds.values():
torch.nn.init.uniform_(ve.weight, -s, s)