tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2025-12-06 04:12:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add back comment

Browse Source
This commit is contained in:
Kian Kyars 2025-11-23 08:09:28 -07:00
parent 1d719a7c94
commit 2f4f20862d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/tok_train.py
View File

@ -77,7 +77,7 @@ vocab_size = tokenizer.get_vocab_size()
special_set = set(tokenizer.get_special_tokens()) special_set = set(tokenizer.get_special_tokens())
token_strings = [tokenizer.decode([token_id]) for token_id in range(vocab_size)] token_strings = [tokenizer.decode([token_id]) for token_id in range(vocab_size)]
token_bytes = [] token_bytes = []
for token_str in token_strings: for token_str in token_strings: # the Python string representation of this token
if token_str in special_set: if token_str in special_set:
token_bytes.append(0) # special characters are not counted token_bytes.append(0) # special characters are not counted
else: else: