📦 deps(thirdparty): update snapshots

2026-06-21 16:02:37 +00:00
parent 3d137606c0
commit c4c6a41c21
305 changed files with 30187 additions and 322 deletions
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Aggregate completed ablation results from the per-instance data disk to durable storage.
+# Idempotent (cp -f overwrites, so a retry result overwrites an epoch-1-failure snapshot).
+#
+# Override DATA_DIR / DURABLE_DIR per your platform profile (profiles/<platform>.md §8). Defaults = AutoDL.
+#
+# Usage: bash aggregate_to_fs.sh   (run on each instance after its queue completes)
+#
+# This is a SAFETY NET — run_one.sh already auto-syncs per ablation. Use it when an auto-sync failed,
+# an older run_one lacked it, or as a final pass before releasing an instance.
+set -u
+
+DATA_DIR="${DATA_DIR:-/root/autodl-tmp}"
+DURABLE_DIR="${DURABLE_DIR:-/root/autodl-fs}"
+
+FS_BASE="$DURABLE_DIR/final_ckpts"
+LOCAL_CKPT_BASE="$DATA_DIR/checkpoints"
+LOCAL_LOG_BASE="$DATA_DIR/runs/logs"
+
+mkdir -p "$FS_BASE"
+
+count=0
+fail=0
+for d in "$LOCAL_CKPT_BASE"/*/; do
+    [ -d "$d" ] || continue
+    name=$(basename "$d")
+
+    # Skip an ablation that never reached epoch 1 (no metrics written).
+    if [ ! -f "$d/best_metrics.json" ]; then
+        echo "SKIP $name (no best_metrics.json)"
+        continue
+    fi
+
+    FS_DIR="$FS_BASE/$name"
+    # GATE on the copy result — never echo OK unconditionally. A full / inode-exhausted durable FS
+    # makes mkdir/cp fail silently; an unconditional "OK" would lie (references/gotchas_universal.md,
+    # silent-sync; principle #3). Verify best.pth landed before counting it.
+    if mkdir -p "$FS_DIR" && cp -f "$d/best.pth" "$FS_DIR/" && [ -f "$FS_DIR/best.pth" ]; then
+        cp -f "$d/best_metrics.json" "$FS_DIR/" 2>/dev/null || true
+        cp -rf "$d/protocol" "$FS_DIR/" 2>/dev/null || true
+        cp -f "$LOCAL_LOG_BASE/$name.log" "$FS_DIR/" 2>/dev/null || true
+        echo "OK $name"
+        count=$((count+1))
+    else
+        echo "!! FAIL $name — durable copy did not land (check 'df -i $DURABLE_DIR'). Data-disk copy is source-of-truth."
+        fail=$((fail+1))
+    fi
+done
+
+echo
+echo "=== Aggregated $count ablations to $FS_BASE ($fail failed) ==="
+echo "Total dirs on durable FS now: $(find "$FS_BASE" -mindepth 1 -maxdepth 1 -type d | wc -l)"
+df -h "$FS_BASE" | tail -1
+df -i "$FS_BASE" | tail -1
+[ "$fail" -eq 0 ] || exit 1