From 39cee6fc76598005b42fafb93c1fba6b374d80d0 Mon Sep 17 00:00:00 2001
From: Hayden Free <hayden@damoursystems.com>
Date: Sun, 26 Apr 2026 00:02:19 -0400
Subject: [PATCH] Exclude dataset shards from HF uploads

---
 runs/runpod/d12.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/runs/runpod/d12.sh b/runs/runpod/d12.sh
index aea99d9a..2962701a 100755
--- a/runs/runpod/d12.sh
+++ b/runs/runpod/d12.sh
@@ -59,8 +59,10 @@ cleanup() {
   if [ "$rc" -eq 0 ]; then
     echo "[runner] success — final upload to $HF_REPO"
     if [ -d "$NANOCHAT_BASE_DIR" ]; then
+      # Skip the climbmix dataset shards (~2GB of public data, not model artifacts)
       hf upload "$HF_REPO" "$NANOCHAT_BASE_DIR" . \
-        --repo-type model --commit-message "final rc=0 $TS" || \
+        --repo-type model --commit-message "final rc=0 $TS" \
+        --exclude "base_data_climbmix/**" --exclude "wandb/**" || \
         echo "[runner] WARN: final upload failed"
     fi
   else
@@ -77,7 +79,8 @@ cleanup() {
     if [ "$UPLOAD_FAILURE_CACHE" = "1" ] && [ -d "$NANOCHAT_BASE_DIR" ]; then
       echo "[runner] UPLOAD_FAILURE_CACHE=1 — also dumping partial cache (may be slow)"
       hf upload "$HF_REPO" "$NANOCHAT_BASE_DIR" "_failures/${TS}-rc${rc}/cache" \
-        --repo-type model --commit-message "failure rc=$rc cache $TS" || true
+        --repo-type model --commit-message "failure rc=$rc cache $TS" \
+        --exclude "base_data_climbmix/**" --exclude "wandb/**" || true
     fi
     echo "[runner] failure artifacts: https://huggingface.co/$HF_REPO/tree/main/_failures/${TS}-rc${rc}"
   fi
@@ -148,7 +151,9 @@ echo "[runner] === FA3 PROBE END ==="
     if [ -d "$NANOCHAT_BASE_DIR" ]; then
       hf upload "$HF_REPO" "$NANOCHAT_BASE_DIR" . \
         --repo-type model \
-        --commit-message "checkpoint $(date -Iseconds)" >> /workspace/backup.log 2>&1 || true
+        --commit-message "checkpoint $(date -Iseconds)" \
+        --exclude "base_data_climbmix/**" --exclude "wandb/**" \
+        >> /workspace/backup.log 2>&1 || true
     fi
   done
 ) &