This commit is contained in:
Shizhe Diao 2025-10-22 21:13:34 -07:00
parent 4b62a8b00c
commit 66a92fc293
4 changed files with 2 additions and 4 deletions

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ rustbpe/target/
dev-ignore/ dev-ignore/
report.md report.md
eval_bundle/ eval_bundle/
*.parquet

View File

@ -1,6 +1,3 @@
# pip install -U "huggingface_hub[hf_transfer]"
# export HF_HUB_ENABLE_HF_TRANSFER=1
python download_climbmix.py python download_climbmix.py
python detokenize_climbmix.py --input_folder climbmix_small --output_folder ./ python detokenize_climbmix.py --input_folder climbmix_small --output_folder ./
rm -rf climbmix_small rm -rf climbmix_small