small change to doc string at top of tok_train.py (#402)

This commit is contained in:
helloaidank 2025-12-31 20:57:26 +00:00 committed by GitHub
parent 8c89661465
commit 389d019a0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -1,5 +1,5 @@
"""
Train a tokenizer using the HuggingFace Tokenizers library.
Train a tokenizer using our own BPE Tokenizer library.
In the style of GPT-4 tokenizer.
"""
import os