From fc23c1aa71823f78ef804f82c54dccf938815bb7 Mon Sep 17 00:00:00 2001 From: Shizhe Diao Date: Mon, 20 Oct 2025 11:11:31 -0700 Subject: [PATCH] use the same tokenizer --- nanochat/checkpoint_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nanochat/checkpoint_manager.py b/nanochat/checkpoint_manager.py index bb8af86..0036e7b 100644 --- a/nanochat/checkpoint_manager.py +++ b/nanochat/checkpoint_manager.py @@ -82,7 +82,7 @@ def build_model(checkpoint_dir, step, device, phase): else: model.train() # Load the Tokenizer - tokenizer_name = meta_data["tokenizer_name"] + tokenizer_name = meta_data["tokenizer_name"] if "tokenizer_name" in meta_data else "tokenizer" print(f"Loading tokenizer: {tokenizer_name}") tokenizer = get_tokenizer(tokenizer_name) # Sanity check: compatibility between model and tokenizer