tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2025-12-06 04:12:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 3b372875c1 into 4a87a0d19f

Browse Source
This commit is contained in:
Sofie Van Landeghem 2025-11-18 03:15:02 +03:00 committed by GitHub
commit 72c0925c88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
.venv/ .venv/
__pycache__/ __pycache__/
*.pyc *.pyc
*.so
rustbpe/target/ rustbpe/target/
dev-ignore/ dev-ignore/
report.md report.md

2
nanochat/tokenizer.py
View File

@ -149,7 +149,7 @@ class HuggingFaceTokenizer:
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Tokenizer based on rustbpe + tiktoken combo # Tokenizer based on rustbpe + tiktoken combo
import pickle import pickle
import rustbpe from nanochat import rustbpe
import tiktoken import tiktoken
class RustBPETokenizer: class RustBPETokenizer:

14
pyproject.toml
View File

@ -23,7 +23,7 @@ requires = ["maturin>=1.7,<2.0"]
build-backend = "maturin" build-backend = "maturin"
[tool.maturin] [tool.maturin]
module-name = "rustbpe" module-name = "nanochat.rustbpe"
bindings = "pyo3" bindings = "pyo3"
python-source = "." python-source = "."
manifest-path = "rustbpe/Cargo.toml" manifest-path = "rustbpe/Cargo.toml"
@ -67,9 +67,15 @@ cpu = [
gpu = [ gpu = [
"torch>=2.8.0", "torch>=2.8.0",
] ]
[tool.uv] [tool.uv]
conflicts = [ cache-keys = [
{ file = "pyproject.toml" },
{ file = "rustbpe/src/**/*.rs" },
{ file = "rustbpe/Cargo.toml" },
{ file = "rustbpe/Cargo.lock" }
]
conflicts = [
[ [
{ extra = "cpu" }, { extra = "cpu" },
{ extra = "gpu" }, { extra = "gpu" },