diff --git a/.gitignore b/.gitignore index 4a87b23..5ff32c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .venv/ __pycache__/ *.pyc +*.so rustbpe/target/ dev-ignore/ report.md diff --git a/nanochat/tokenizer.py b/nanochat/tokenizer.py index 880f854..9df5549 100644 --- a/nanochat/tokenizer.py +++ b/nanochat/tokenizer.py @@ -149,7 +149,7 @@ class HuggingFaceTokenizer: # ----------------------------------------------------------------------------- # Tokenizer based on rustbpe + tiktoken combo import pickle -import rustbpe +from nanochat import rustbpe import tiktoken class RustBPETokenizer: diff --git a/pyproject.toml b/pyproject.toml index 3d03c4b..e06995d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ requires = ["maturin>=1.7,<2.0"] build-backend = "maturin" [tool.maturin] -module-name = "rustbpe" +module-name = "nanochat.rustbpe" bindings = "pyo3" python-source = "." manifest-path = "rustbpe/Cargo.toml" @@ -67,9 +67,15 @@ cpu = [ gpu = [ "torch>=2.8.0", ] - -[tool.uv] -conflicts = [ + +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "rustbpe/src/**/*.rs" }, + { file = "rustbpe/Cargo.toml" }, + { file = "rustbpe/Cargo.lock" } +] +conflicts = [ [ { extra = "cpu" }, { extra = "gpu" },