tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2026-02-13 05:59:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 9f801891a9 into 8630d32be4

Browse Source
This commit is contained in:
aioaneid 2026-01-27 14:44:34 +01:00 committed by GitHub
commit 59883f3518
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runs/run1000.sh
View File

@ -19,7 +19,7 @@ python -m nanochat.report reset
curl -L -o $NANOCHAT_BASE_DIR/identity_conversations.jsonl https://karpathy-public.s3.us-west-2.amazonaws.com/identity_conversations.jsonl
# train tokenizer on ~4B characters and kick off download of the rest for pretraining
python -m nanochat.dataset -n 16
python -m nanochat.dataset -n 21
# start downloading the rest of the shards for a total of 1200 (see below why 1200)
python -m nanochat.dataset -n 1200 &
# todo: download the rest of it

2
runs/runcpu.sh
View File

@ -24,7 +24,7 @@ if [ -z "$WANDB_RUN" ]; then
fi
# train tokenizer on ~2B characters (~34 seconds on my MacBook Pro M3 Max)
python -m nanochat.dataset -n 8
python -m nanochat.dataset -n 11
python -m scripts.tok_train --max-chars=2000000000
python -m scripts.tok_eval

9
runs/speedrun.sh
View File

@ -50,10 +50,11 @@ python -m nanochat.report reset
# Download the first ~2B characters of pretraining dataset
# look at dev/repackage_data_reference.py for details on how this data was prepared
# each data shard is ~250M chars
# so we download 2e9 / 250e6 = 8 data shards at this point
# each shard is ~100MB of text (compressed), so this is about ~800MB of data on disk
python -m nanochat.dataset -n 8
# each data shard is ~250M chars, but due to the `doc_cap` only ~200M chars are used.
# The last shard is considered to be eval, so we download 2e9 / 200e6 + 1 = 11 data
# shards at this point
# each shard is ~90MB of text (compressed), so this is about ~1GB of data on disk
python -m nanochat.dataset -n 11
# Immediately also kick off downloading more shards in the background while tokenizer trains
# See comment below for why 370 is the right number here
python -m nanochat.dataset -n 370 &