Upload eden_hf_upload.py with huggingface_hub

eden_hf_upload.py

@@ -0,0 +1,458 @@
+"""
+Project EDEN - Hugging Face Upload Master Script
+Applies all 6 refinements:
+  1. Hardware transparency (1080 Ti / Xeon W-2125)
+  2. E2AM Phase mapping per model
+  3. Phase 1 Zero-Overhead Initialization highlight
+  4. Standardized Green Delta table in every README
+  5. YAML tags with co2_eq_emissions + dataset_size
+  6. Citation section in Main Repo
+"""
+
+import os
+import json
+import glob
+import math
+from huggingface_hub import HfApi, create_repo, upload_file
+
+# ─── CONFIG ──────────────────────────────────────────────────────────────────
+HF_TOKEN   = os.environ.get("HF_TOKEN", "")
+HF_USER    = "Shanmuk4622"          # HF username (no org found, uploading under user)
+HF_ORG     = HF_USER               # use user namespace
+BASE_DIR   = os.path.dirname(os.path.abspath(__file__))
+DRY_RUN    = False  # Live upload
+
+api = HfApi(token=HF_TOKEN)
+
+# ─── HARDWARE PROFILE ────────────────────────────────────────────────────────
+HARDWARE = {
+    "gpu":  "NVIDIA GeForce GTX 1080 Ti (11 GB VRAM, 250 W TDP)",
+    "cpu":  "Intel Xeon W-2125 (4 cores / 8 threads @ 4.00 GHz)",
+    "ram":  "63.66 GB System RAM",
+    "os":   "Windows 10",
+}
+
+# ─── E2AM PHASE MAP ──────────────────────────────────────────────────────────
+# Maps folder -> technique label for README
+PHASE_MAP = {
+    "test1": "Phase 2 – Progressive Unfreezing + AMP (E2AM SOTA)",
+    "test2": "Baseline – Standard Full Training (Reference Study)",
+    "test3": "Phase 2 – EDEN Classic Energy-Aware Sparse Training",
+}
+
+PHASE_DETAIL = {
+    "test1": (
+        "**Phase 1 – Zero-Overhead Initialization:** Dataset pre-loaded into pinned "
+        "System RAM to eliminate disk I/O power spikes.\n\n"
+        "**Phase 2 – Progressive Unfreezing:** Backbone frozen for the first "
+        "`E_unfreeze` epochs (only the classification head trains). At `E_unfreeze`, "
+        "all layers are unfrozen and the learning rate is decayed. "
+        "Gradient accumulation over N micro-batches simulates large batch sizes "
+        "without proportional VRAM cost, slashing power-draw spikes.\n\n"
+        "**AMP (Automated Mixed Precision):** `torch.cuda.amp.autocast()` halves "
+        "GPU memory bandwidth, reducing energy per backward pass.\n\n"
+        "**Sparse Regularisation:** L1 penalty `λ·Σ|W|` applied to trainable "
+        "weights, driving dead neurons to zero and enabling future pruning."
+    ),
+    "test2": (
+        "Standard full fine-tuning used as the **Brute-Force Baseline** for "
+        "energy comparison. All layers trained from epoch 1 with a fixed learning "
+        "rate and no gradient accumulation. Included for transparent EAG benchmarking."
+    ),
+    "test3": (
+        "**Phase 1 – Zero-Overhead Initialization:** Dataset cached in System RAM.\n\n"
+        "**Phase 2 – EDEN Classic:** Energy-aware training loop on classic CNN "
+        "architectures. Applies the same EAG early-exit criterion "
+        "(`EAG < γ_EAG` for 3 consecutive epochs → terminate), L1 sparsity "
+        "penalty, and AMP to architectures like ResNet, VGG, AlexNet, DenseNet, "
+        "InceptionV3, and UNet."
+    ),
+}
+
+# ─── DATASET META ────────────────────────────────────────────────────────────
+DATASET_META = {
+    "CIFAR-10":           {"size": "60,000 images – 10 classes (32×32 px)", "hf_name": "cifar10"},
+    "CIFAR-100":          {"size": "60,000 images – 100 classes (32×32 px)", "hf_name": "cifar100"},
+    "Custom-ImageNet300": {"size": "~450,000 images – 300 classes (224 px)", "hf_name": "imagenet"},
+    "unknown":            {"size": "N/A", "hf_name": "unknown"},
+}
+
+# CO2: 0.475 kg CO2e per kWh (global average grid factor)
+KG_CO2_PER_KWH = 0.000000475   # per Joule
+
+# ─── HELPERS ─────────────────────────────────────────────────────────────────
+def parse_name(filename):
+    fn = filename.lower().replace("\\", "/")
+    dataset = "unknown"
+    arch    = "unknown"
+    if   "cifar100" in fn:  dataset = "CIFAR-100"
+    elif "cifar10"  in fn:  dataset = "CIFAR-10"
+    elif "imagenet" in fn:  dataset = "Custom-ImageNet300"
+    if   "efficientnet" in fn: arch = "EfficientNetV2"
+    elif "convnext"     in fn: arch = "ConvNeXtV2"
+    elif "mobilevit"    in fn: arch = "MobileViTv3"
+    elif "resnet50"     in fn: arch = "ResNet50"
+    elif "resnet18"     in fn: arch = "ResNet18"
+    elif "vgg16"        in fn: arch = "VGG16"
+    elif "alexnet"      in fn: arch = "AlexNet"
+    elif "inception"    in fn: arch = "InceptionV3"
+    elif "densenet"     in fn: arch = "DenseNet121"
+    elif "unet"         in fn: arch = "UNet"
+    return arch, dataset
+
+def joules_to_co2(joules):
+    kwh = joules / 3_600_000
+    return kwh * 0.475  # kg CO2e
+
+def folder_to_phase_label(folder):
+    return {"test1": "SOTA Optimized", "test2": "Baseline", "test3": "EDEN Classic"}.get(folder, folder)
+
+# ─── LOAD STATS ──────────────────────────────────────────────────────────────
+with open(os.path.join(BASE_DIR, "results_summary.json")) as f:
+    results = json.load(f)
+
+stats_map = {}
+for r in results:
+    arch, dataset = parse_name(r["file"])
+    folder = r["folder"]
+    key = f"{folder}_{arch}_{dataset}"
+    if key not in stats_map or (r["energy"] > 0 and stats_map[key]["energy"] == 0):
+        stats_map[key] = r
+
+# Build baseline map (ResNet50 from test2 per dataset)
+baselines = {}
+for key, v in stats_map.items():
+    folder, *rest = key.split("_")
+    arch = v.get("arch") or parse_name(v["file"])[0]
+    if folder == "test2":
+        _, ds = parse_name(v["file"])
+        if ds not in baselines:
+            baselines[ds] = v
+        # prefer ResNet50
+        if parse_name(v["file"])[0] == "ResNet50":
+            baselines[ds] = v
+
+# ─── COLLECT ALL MODELS ──────────────────────────────────────────────────────
+pth_files = glob.glob(os.path.join(BASE_DIR, "**/*.pth"), recursive=True)
+models = []
+for pth in pth_files:
+    rel = os.path.relpath(pth, BASE_DIR)
+    parts = rel.split(os.sep)
+    folder = parts[0]
+    arch, dataset = parse_name(rel)
+    key = f"{folder}_{arch}_{dataset}"
+    stat = stats_map.get(key, {})
+    models.append({
+        "pth": rel, "arch": arch, "dataset": dataset,
+        "folder": folder,
+        "accuracy": stat.get("accuracy", 0),
+        "energy":   stat.get("energy",   0),
+        "time":     stat.get("time",     0),
+        "csv":      stat.get("file",    "N/A"),
+    })
+
+# ─── README GENERATOR ────────────────────────────────────────────────────────
+def build_readme(model):
+    arch     = model["arch"]
+    dataset  = model["dataset"]
+    folder   = model["folder"]
+    acc      = model["accuracy"]
+    energy   = model["energy"]
+    t        = model["time"]
+    phase    = folder_to_phase_label(folder)
+    ds_meta  = DATASET_META.get(dataset, DATASET_META["unknown"])
+    co2      = joules_to_co2(energy) if energy else 0
+
+    baseline = baselines.get(dataset, {})
+    b_acc    = baseline.get("accuracy", 0)
+    b_energy = baseline.get("energy",   0)
+    b_arch   = parse_name(baseline.get("file",""))[0] if baseline else "Baseline"
+
+    # Green Delta
+    if b_energy and energy:
+        energy_savings_pct = (b_energy - energy) / b_energy * 100
+        d_acc = acc - b_acc
+        d_j   = energy - b_energy
+        eag   = d_acc / d_j if d_j != 0 else float("nan")
+        eag_str     = f"{eag:.4e}"
+        savings_str = f"{energy_savings_pct:.2f}%"
+        acc_delta   = f"{d_acc*100:+.2f}%"
+    else:
+        energy_savings_pct = 0
+        eag_str     = "N/A"
+        savings_str = "N/A"
+        acc_delta   = "N/A"
+
+    # YAML tags
+    arch_tag = arch.lower().replace(" ","")
+    yaml_co2 = f"{co2:.4f}" if co2 else "0"
+
+    yaml = f"""---
+language: en
+license: apache-2.0
+tags:
+- image-classification
+- green-ai
+- energy-efficiency
+- computer-vision
+- {arch_tag}
+- eden-framework
+- e2am
+- sustainable-ai
+datasets:
+- {ds_meta['hf_name']}
+metrics:
+- accuracy
+co2_eq_emissions:
+  emissions: {yaml_co2}
+  unit: kg
+  source: Estimated via CodeCarbon (grid factor 0.475 kg CO2e/kWh)
+  hardware_used: NVIDIA GeForce GTX 1080 Ti
+dataset_info:
+  dataset_size: "{ds_meta['size']}"
+---"""
+
+    # Technique section
+    technique = PHASE_DETAIL.get(folder, "Standard training.")
+
+    # Green Delta Table
+    green_table = f"""| Metric | {b_arch} Baseline | **{arch} (EDEN)** | Δ |
+|---|---|---|---|
+| Accuracy | {b_acc:.4f} | **{acc:.4f}** | `{acc_delta}` |
+| Total Energy (J) | {b_energy:,.0f} | **{energy:,.0f}** | `{savings_str} saved` |
+| CO₂ Emissions (kg) | {joules_to_co2(b_energy):.4f} | **{co2:.4f}** | — |
+| **EAG Score** | — | **{eag_str}** | ΔAcc/ΔJoules |"""
+
+    cite = f"""## Cite This Research
+If you use this model, please cite the **EDEN / E2AM Framework**:
+
+```bibtex
+@misc{{eden2025,
+  title     = {{Project EDEN: Energy-Driven Evolution of Networks}},
+  author    = {{EDEN Research Team}},
+  year      = {{2025}},
+  note      = {{Hugging Face Organization: ProjectEDEN}},
+  url       = {{https://huggingface.co/{HF_ORG}}}
+}}
+```"""
+
+    readme = f"""{yaml}
+
+# EDEN-{arch}-{dataset} ��� *{phase}*
+
+> **Primary KPI:** EAG (Energy-to-Accuracy Gradient) = `{eag_str}` ΔAcc/ΔJoules
+
+## Abstract
+This model is part of **Project EDEN (Energy-Driven Evolution of Networks)**, implementing the **E2AM (Energy Efficient Advanced Model)** Framework. The goal is to shift AI benchmarking from pure accuracy to *Green SOTA* — maximizing predictive power per Joule consumed.
+
+**Applied Technique:** {PHASE_MAP.get(folder, phase)}
+
+## Profiling Environment
+| Component | Specification |
+|---|---|
+| **GPU** | {HARDWARE['gpu']} |
+| **CPU** | {HARDWARE['cpu']} |
+| **RAM** | {HARDWARE['ram']} |
+| **OS**  | {HARDWARE['os']} |
+| **Dataset** | {dataset} — {ds_meta['size']} |
+
+## 🟢 Green Delta Table
+*Comparing this model against the reference baseline (ResNet-50 equivalent)*
+
+{green_table}
+
+> A **positive EAG** means this model learns more per Joule than the baseline.
+> A **negative EAG** indicates a trade-off where higher accuracy required more energy investment.
+
+## E2AM Algorithm — Applied Phases
+
+{technique}
+
+## Training Statistics
+| Metric | Value |
+|---|---|
+| Final Accuracy | {acc:.4f} ({acc*100:.2f}%) |
+| Total Energy Consumed | {energy:,.0f} J ({energy/3_600_000:.4f} kWh) |
+| Training Time | {t:,.0f} s ({t/3600:.2f} hrs) |
+| Estimated CO₂ | {co2:.4f} kg CO₂e |
+| Training Log | `{model['csv']}` |
+
+{cite}
+"""
+    return readme
+
+# ─── MAIN FRAMEWORK README ───────────────────────────────────────────────────
+def build_main_repo_readme():
+    py_scripts = [os.path.relpath(p, BASE_DIR) for p in
+                  glob.glob(os.path.join(BASE_DIR, "**/*.py"), recursive=True)
+                  if any(k in p for k in ["Algo_", "eden_", "mobilevit_model"])]
+
+    scripts_md = "\n".join(f"- `{s}`" for s in sorted(py_scripts))
+
+    return f"""---
+language: en
+license: apache-2.0
+tags:
+- green-ai
+- energy-efficiency
+- e2am
+- eden-framework
+- sustainable-ai
+- image-classification
+---
+
+# EDEN-Core-Scripts — E2AM Framework Repository
+
+> **Project EDEN (Energy-Driven Evolution of Networks)** — The complete algorithmic
+> toolkit for Green SOTA image classification research.
+
+## Why EDEN?
+As deep learning models scale exponentially, the carbon footprint of training has
+reached unsustainable levels. Project EDEN introduces the **EAG
+(Energy-to-Accuracy Gradient)** as the primary KPI — shifting the paradigm from
+chasing raw accuracy to optimising *Green SOTA*.
+
+## Profiling Environment
+| Component | Specification |
+|---|---|
+| **GPU** | {HARDWARE['gpu']} |
+| **CPU** | {HARDWARE['cpu']} |
+| **RAM** | {HARDWARE['ram']} |
+| **OS**  | {HARDWARE['os']} |
+
+## The E2AM Algorithm — All Three Phases
+
+### Phase 1 — Zero-Overhead Initialization
+Dataset pre-loaded into **pinned System RAM** before training begins.
+This eliminates disk I/O power spikes that would otherwise inflate energy readings
+and distort EAG comparisons between architectures.
+
+### Phase 2 — Two-Stage Energy-Aware Training
+1. **Frozen Head Training** — Only the classification head trains for the first
+   `E_unfreeze` epochs. The backbone consumes no backward-pass energy.
+2. **Progressive Unfreezing** — At epoch `E_unfreeze`, all layers unlock.
+   Learning rate is decayed (`LR × 0.1`) for stable fine-tuning.
+3. **Gradient Accumulation** — Gradients accumulated over N micro-batches,
+   simulating large batch sizes without VRAM spikes.
+4. **AMP (Automated Mixed Precision)** — `torch.cuda.amp.autocast()` halves
+   bandwidth per backward pass.
+5. **Sparse L1 Penalty** — `L_total = CrossEntropy + λ·Σ|W_trainable|`
+6. **EAG Early-Exit** — Training terminates if `EAG < γ_EAG` for 3 consecutive
+   epochs, preventing wasted compute.
+
+### Phase 3 — Hardware-Aware Deployment *(Post-Training)*
+- **Saliency-Energy Pruning** — Filters with lowest `∂Accuracy/∂W ÷ Energy_cost`
+  are pruned.
+- **INT8 Quantization** — Weights converted for edge-deployment readiness.
+- **Dynamic Depth Routing** — Simple images bypass the middle 50 % of layers
+  via residual skip connections, slashing inference energy.
+
+## EAG — The Expert KPI
+```
+EAG = ΔAccuracy / ΔJoules
+```
+EAG allows apples-to-apples comparison of any two models regardless of
+architecture family. A higher EAG = more learning per unit of carbon footprint.
+
+## Scripts in This Repository
+{scripts_md}
+
+## Cite This Research
+```bibtex
+@misc{{eden2025,
+  title     = {{Project EDEN: Energy-Driven Evolution of Networks}},
+  author    = {{EDEN Research Team}},
+  year      = {{2025}},
+  note      = {{Hugging Face Organization: ProjectEDEN}},
+  url       = {{https://huggingface.co/{HF_ORG}}}
+}}
+```
+"""
+
+# ─── OUTPUT / UPLOAD ─────────────────────────────────────────────────────────
+OUT_DIR = os.path.join(BASE_DIR, "hf_readmes")
+os.makedirs(OUT_DIR, exist_ok=True)
+
+# 1. Main repo README
+main_readme = build_main_repo_readme()
+main_readme_path = os.path.join(OUT_DIR, "EDEN-Core-Scripts_README.md")
+with open(main_readme_path, "w", encoding="utf-8") as f:
+    f.write(main_readme)
+print("✓ Main repo README written.")
+
+# 2. Per-model READMEs (deduplicated by repo name)
+generated_repos = set()
+repo_model_map  = {}   # repo_name -> (model, readme_text)
+
+for m in models:
+    if m["arch"] == "unknown" or m["dataset"] == "unknown": continue
+    repo_name = f"EDEN-{m['arch']}-{m['dataset'].replace(' ','-')}"
+    # prefer highest-accuracy model per repo
+    if repo_name not in repo_model_map or m["accuracy"] > repo_model_map[repo_name][0]["accuracy"]:
+        readme_text = build_readme(m)
+        repo_model_map[repo_name] = (m, readme_text)
+
+for repo_name, (m, readme_text) in repo_model_map.items():
+    path = os.path.join(OUT_DIR, f"{repo_name}_README.md")
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(readme_text)
+    print(f"✓ {repo_name} README written.")
+
+print(f"\n{'='*60}")
+print(f"Generated {len(repo_model_map)+1} README files in: {OUT_DIR}")
+
+if not DRY_RUN:
+    print("\nStarting HF upload...")
+
+    # Upload Main Repo README
+    try:
+        create_repo(repo_id=f"{HF_ORG}/EDEN-Core-Scripts", token=HF_TOKEN,
+                    repo_type="model", exist_ok=True, private=False)
+        upload_file(path_or_fileobj=main_readme_path,
+                    path_in_repo="README.md",
+                    repo_id=f"{HF_ORG}/EDEN-Core-Scripts",
+                    token=HF_TOKEN, repo_type="model")
+        # Upload all .py scripts
+        for py in glob.glob(os.path.join(BASE_DIR, "**/*.py"), recursive=True):
+            rel = os.path.relpath(py, BASE_DIR)
+            if any(k in rel for k in ["Algo_","eden_","mobilevit_model"]):
+                upload_file(path_or_fileobj=py,
+                            path_in_repo=rel.replace("\\","/"),
+                            repo_id=f"{HF_ORG}/EDEN-Core-Scripts",
+                            token=HF_TOKEN, repo_type="model")
+        print("✓ Uploaded EDEN-Core-Scripts")
+    except Exception as e:
+        print(f"✗ Core-Scripts error: {e}")
+
+    # Upload per-model repos
+    for repo_name, (m, readme_text) in repo_model_map.items():
+        try:
+            create_repo(repo_id=f"{HF_ORG}/{repo_name}", token=HF_TOKEN,
+                        repo_type="model", exist_ok=True, private=False)
+            readme_path = os.path.join(OUT_DIR, f"{repo_name}_README.md")
+            upload_file(path_or_fileobj=readme_path,
+                        path_in_repo="README.md",
+                        repo_id=f"{HF_ORG}/{repo_name}",
+                        token=HF_TOKEN, repo_type="model")
+            # Upload weights
+            pth_abs = os.path.join(BASE_DIR, m["pth"])
+            if os.path.exists(pth_abs):
+                upload_file(path_or_fileobj=pth_abs,
+                            path_in_repo=os.path.basename(m["pth"]),
+                            repo_id=f"{HF_ORG}/{repo_name}",
+                            token=HF_TOKEN, repo_type="model")
+            # Upload CSV log
+            if m["csv"] != "N/A":
+                csv_abs = os.path.join(BASE_DIR, m["csv"])
+                if os.path.exists(csv_abs):
+                    upload_file(path_or_fileobj=csv_abs,
+                                path_in_repo=os.path.basename(m["csv"]),
+                                repo_id=f"{HF_ORG}/{repo_name}",
+                                token=HF_TOKEN, repo_type="model")
+            print(f"✓ Uploaded {repo_name}")
+        except Exception as e:
+            print(f"✗ {repo_name} error: {e}")
+
+    print("\nAll uploads complete.")
+else:
+    print("\n[DRY RUN] Set DRY_RUN=False to execute HF uploads.")