Upload folder using huggingface_hub

trash2.py

@@ -0,0 +1,113 @@
+import os
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+
+# --- Configuration ---
+MANIFEST_ROOT = "/root/hf_repo/weights"
+FSX_ROOT = "/fsx_scanline/from_eyeline/koichi/motion-aware-vid-stylization/train_ckpt/Netflix65K"
+DEST_ROOT = "/hf_repo/weights"
+
+def find_copy_tasks():
+    """
+    1. Scans MANIFEST_ROOT for model folders.
+    2. Scans inside those folders for specific .safetensors files.
+    Returns: list of (model_alias, ckpt_folder, filename)
+    """
+    tasks = []
+    
+    if not os.path.exists(MANIFEST_ROOT):
+        print(f"Error: Manifest root {MANIFEST_ROOT} does not exist.")
+        return []
+
+    print(f"Scanning {MANIFEST_ROOT} for specific safetensor files...")
+    
+    try:
+        model_aliases = [d for d in os.listdir(MANIFEST_ROOT) if os.path.isdir(os.path.join(MANIFEST_ROOT, d))]
+    except OSError as e:
+        print(f"Error reading manifest root: {e}")
+        return []
+
+    for alias in model_aliases:
+        alias_path = os.path.join(MANIFEST_ROOT, alias)
+        
+        try:
+            # Find checkpoint folders (e.g. AllShrink_...)
+            checkpoints = [d for d in os.listdir(alias_path) if os.path.isdir(os.path.join(alias_path, d))]
+        except OSError:
+            continue
+            
+        for ckpt_folder in checkpoints:
+            ckpt_full_path = os.path.join(alias_path, ckpt_folder)
+            
+            # --- CRITICAL CHANGE ---
+            # We look inside the folder to see WHICH safetensors file is there
+            try:
+                files_in_ckpt = os.listdir(ckpt_full_path)
+                target_files = [f for f in files_in_ckpt if f.endswith('.safetensors')]
+                
+                if not target_files:
+                    # Optional: warning if a folder is empty
+                    # print(f"Warning: No safetensors found in {alias}/{ckpt_folder}")
+                    pass
+                
+                for filename in target_files:
+                    # We create a task for this SPECIFIC file
+                    tasks.append((alias, ckpt_folder, filename))
+                    
+            except OSError:
+                continue
+            
+    print(f"Found {len(tasks)} specific files to copy.")
+    return tasks
+
+def process_copy(task):
+    """
+    Copies a SINGLE file from FSX to Destination using rclone copyto
+    """
+    alias, ckpt_folder, filename = task
+    
+    # 1. Source File (Flat FSX structure)
+    # /fsx/.../Netflix65K/{ckpt_folder}/{filename}
+    src_file_path = os.path.join(FSX_ROOT, ckpt_folder, filename)
+    
+    # 2. Dest File (Nested structure)
+    # /hf_repo/weights/{alias}/{ckpt_folder}/{filename}
+    dest_file_path = os.path.join(DEST_ROOT, alias, ckpt_folder, filename)
+    
+    # Check if the specific file exists on FSX source
+    if not os.path.exists(src_file_path):
+        print(f"[{alias}] MISSING SOURCE: {filename} (Expected at {src_file_path})")
+        return
+
+    # 3. Use 'copyto' for single files
+    # This is more precise than 'copy'
+    cmd = [
+        "rclone", "copyto",
+        src_file_path,
+        dest_file_path,
+        "--transfers", "4",
+        "--progress" 
+    ]
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        
+        if result.returncode == 0:
+            print(f"[{alias}] SUCCESS: {filename}")
+        else:
+            print(f"[{alias}] ERROR copying {filename}: {result.stderr.strip()}")
+            
+    except Exception as e:
+        print(f"[{alias}] CRASHED on {filename}: {e}")
+
+if __name__ == "__main__":
+    all_tasks = find_copy_tasks()
+    
+    # Parallel execution
+    # Since we are copying individual large files, 4-8 workers is usually optimal
+    if all_tasks:
+        print(f"Starting parallel copy with {min(len(all_tasks), 8)} workers...")
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            executor.map(process_copy, all_tasks)
+            
+    print("All tasks completed.")

wan21_480P/AllShrink_Uniform256_Inpaint15Other5_14BV3_LightSparseConcatPosiMaxPool3dV3196_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-35000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e54dc2d7c7a1468ba5337f77946b23e14a0360ed6d59cde1bb0691b33aa4acd
+size 39677671844

wan21_720P/AllShrink_720P_14BInp10POd_LightSparseConcatPosiMaxPool3dV2_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4_MinTraj256_W1000/step-31000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9384969028ff625f2e603eeb45ea3df066ede1812d383bf8a98a1b757dd5062
+size 39676881004

wan22_480P/AllShrink_HighNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-40000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e3d4cbaacf72a49cbf8ff0bd1951dae1656b5c6004428fe6329f239a4a9b0c9
+size 39677234564

wan22_480P/AllShrink_HighNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-50000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c61a76dabcc219e07e2ac24577732b7aab547da814ce5c47b65a065b9b4f02b
+size 39677234564

wan22_480P/AllShrink_HighNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-60000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:188fedd0be2dfd00c61e61c42150336c79633c895390c8d568833155ac6adc00
+size 39677234564

wan22_480P/AllShrink_HighNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-72000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb031a295739ec385aee871dcd53de6e3edf06e844021491a0c885ecdcc61340
+size 39677234564

wan22_480P/AllShrink_LowNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-25000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e475618be89c83b1e740e534197e43894ffebb730fa8e3e62bb3cd7ba0294b59
+size 39677234564

wan22_480P/AllShrink_LowNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-30000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e30bf7d0c20a9f96c1197140d46a288608a1453e8805c3079de9519f476ba373
+size 39677234564

wan22_480P/AllShrink_LowNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-35000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d306bd2ede4863f48b02595af65b7307bd0a19df5a83443d76e39d362d34c6c0
+size 39677234564

wan22_480P/AllShrink_LowNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-41000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c359703e4999562abb595153ccf5cfa6c289bf3f3b05c11448e46d9d56e2588
+size 39677234564

wan22_480P_depth/AllShrink_HighNoise14B22_WeightDecay3_Depth50_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-50000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c7260ed9cca333da8571e262238d330768a752ea2dfe2b2051765e42fd5f2f5
+size 39677234564

wan22_480P_depth/AllShrink_HighNoise14B22_WeightDecay3_Depth50_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-60000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e74f0acb534c0db9310a9ccd6de1c3ff564fc74aab041f8742708a6c446d5083
+size 39677234564

wan22_480P_depth/AllShrink_HighNoise14B22_WeightDecay3_Depth50_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-71000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa4982acd728f6bff7fdb8accd01ef31a04d8b942090ef9cd028e6919237f12b
+size 39677234564

wan22_480P_depth/AllShrink_LowNoise14B22_Depth50_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-27000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50bdab27e0ebe13a7e5ed153839b6acdb55020407b2a045189e479b5f60f9a19
+size 39677234564

wan22_720P/AllShrink_720PHighNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-75000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8976eaf82ec53b9b37a241499c8e09a21be165626594a34608f365c05f5075e4
+size 39677234564

wan22_720P/AllShrink_720PLowNoise14B22_WeightDecay3_Uniform256_Inpaint15Other7_14BV3_LightSparseConcatPosiMaxPool3dV3156_ConcatProj_NoDecomp_RefTimeNew_SelfQKOVFFN_Only49_1e5_AW3_DS_Ref4/step-40000-epoch-0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6771b4db4cd0bd893bed3013baab0d67b9c4ee6859dccd0af2761b489a1cd7ed
+size 39677234564