diff --git a/pyproject.toml b/pyproject.toml
index 0d3e853..cf392b5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,8 +70,6 @@ conflicts = [
         { extra = "gpu" },
     ],
 ]
-# rebuild if rustbpe is modified
-cache-keys = [{ file = "pyproject.toml" }, {file = "rustbpe/**"}]
 
 [tool.uv.workspace]
 members = ["rustbpe"]
diff --git a/rustbpe/pyproject.toml b/rustbpe/pyproject.toml
index b71ed05..cd15f32 100644
--- a/rustbpe/pyproject.toml
+++ b/rustbpe/pyproject.toml
@@ -15,3 +15,11 @@ module-name = "rustbpe"
 bindings = "pyo3"
 manifest-path = "Cargo.toml"
 features = ["pyo3/extension-module"]
+
+[tool.uv]
+cache-keys = [
+    { file = "pyproject.toml" },
+    { file = "src/**/*.rs" },
+    { file = "Cargo.toml" },
+    { file = "Cargo.lock" }
+]