tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2025-12-06 04:12:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 3b372875c1 into 4a87a0d19f

Browse Source
This commit is contained in:
Sofie Van Landeghem 2025-11-18 03:15:02 +03:00 committed by GitHub
commit 72c0925c88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
.venv/
__pycache__/
*.pyc
*.so
rustbpe/target/
dev-ignore/
report.md

2
nanochat/tokenizer.py
View File

@ -149,7 +149,7 @@ class HuggingFaceTokenizer:
# -----------------------------------------------------------------------------
# Tokenizer based on rustbpe + tiktoken combo
import pickle
import rustbpe
from nanochat import rustbpe
import tiktoken
class RustBPETokenizer:

8
pyproject.toml
View File

@ -23,7 +23,7 @@ requires = ["maturin>=1.7,<2.0"]
build-backend = "maturin"
[tool.maturin]
module-name = "rustbpe"
module-name = "nanochat.rustbpe"
bindings = "pyo3"
python-source = "."
manifest-path = "rustbpe/Cargo.toml"
@ -69,6 +69,12 @@ gpu = [
]
[tool.uv]
cache-keys = [
{ file = "pyproject.toml" },
{ file = "rustbpe/src/**/*.rs" },
{ file = "rustbpe/Cargo.toml" },
{ file = "rustbpe/Cargo.lock" }
]
conflicts = [
[
{ extra = "cpu" },