diff --git a/runs/speedrun.sh b/runs/speedrun.sh index 8fff5640..ef4fa00d 100644 --- a/runs/speedrun.sh +++ b/runs/speedrun.sh @@ -58,8 +58,8 @@ python -m nanochat.dataset -n 8 # See comment below for why 370 is the right number here python -m nanochat.dataset -n 370 & DATASET_DOWNLOAD_PID=$! -# train the tokenizer with vocab size 2**16 = 65536 on ~2B characters of data -python -m scripts.tok_train --max-chars=2000000000 --vocab-size=65536 +# train the tokenizer with vocab size 2**15 = 32768 on ~2B characters of data +python -m scripts.tok_train # evaluate the tokenizer (report compression ratio etc.) python -m scripts.tok_eval diff --git a/scripts/tok_train.py b/scripts/tok_train.py index 9c7979d2..480e0e16 100644 --- a/scripts/tok_train.py +++ b/scripts/tok_train.py @@ -14,7 +14,7 @@ from nanochat.dataset import parquets_iter_batched # Parse command line arguments parser = argparse.ArgumentParser(description='Train a BPE tokenizer') -parser.add_argument('--max-chars', type=int, default=10_000_000_000, help='Maximum characters to train on (default: 10B)') +parser.add_argument('--max-chars', type=int, default=2_000_000_000, help='Maximum characters to train on (default: 10B)') parser.add_argument('--doc-cap', type=int, default=10_000, help='Maximum characters per document (default: 10,000)') parser.add_argument('--vocab-size', type=int, default=32768, help='Vocabulary size (default: 32768 = 2^15)') args = parser.parse_args()