tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2026-06-15 10:39:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 46812c4160 into dc54a1a307

Browse Source
This commit is contained in:
Arthur Testard 2026-05-05 08:42:15 +02:00 committed by GitHub
commit 1fc2eb7923
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
nanochat/checkpoint_manager.py
View File

@ -38,6 +38,15 @@ def _patch_missing_keys(model_data, model_config):
if "x0_lambdas" not in model_data:
model_data["x0_lambdas"] = torch.zeros(n_layer)
log0(f"Patching missing x0_lambdas in model data to 0.0")
if "smear_gate.weights" not in model_data:
model_data["smear_gate.weights"] = torch.ones(1, 24)
log0(f"Patching missing smear_gate.weights in model data to 1.0")
if "smear_lambda" not in model_data:
model_data["smear_lambda"] = torch.zeros(1)
log0(f"Patching missing smear_lambda in model data to 0.0")
if "backout_lambda" not in model_data:
model_data["backout_lambda"] = 0.2 * torch.ones(1)
log0(f"Patching missing backout_lambda in model data to 0.2")
def save_checkpoint(checkpoint_dir, step, model_data, optimizer_data, meta_data, rank=0):
if rank == 0: