Delete Other_algorithms

Other_algorithms/Flat_System/PG/pg_evaluation.py

@@ -1,520 +0,0 @@
-# pg_evaluate.py
-import os
-import sys
-import time
-import re
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import torch
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from PG.trainer.pg import PGAgent
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-def compute_jains_fairness(values: np.ndarray) -> float:
-    if len(values) == 0:
-        return 0.0
-    if np.all(values == 0):
-        return 1.0
-    num = (values.sum())**2
-    den = len(values) * (values**2).sum()
-    return num / den
-
-def main():
-    # User parameters
-    MODEL_PATH = "/path/to/project/pg_pennsylvania_10agents_10000eps/logs"
-    DATA_PATH = "/path/to/project/testing/10houses_30days_TEST.csv"
-    DAYS_TO_EVALUATE = 30
-
-    model_path = MODEL_PATH
-    data_path = DATA_PATH
-    days_to_evaluate = DAYS_TO_EVALUATE
-    SOLAR_THRESHOLD = 0.5
-
-    state_match = re.search(r"pg_(oklahoma|colorado|pennsylvania)_", model_path)
-    if not state_match:
-        raise ValueError(
-            "Could not automatically detect the state (oklahoma, colorado, or pennsylvania) "
-            "from the model path. Please ensure your model's parent folder is named correctly, "
-            "e.g., 'pg_oklahoma_...'"
-        )
-    detected_state = state_match.group(1)
-    print(f"--- Detected state: {detected_state.upper()} ---")
-
-    # Env setup
-    env = SolarSys(
-        data_path=data_path,
-        state=detected_state,
-        time_freq="15T"
-    )
-    eval_steps = env.num_steps
-    house_ids = env.house_ids
-    num_agents = env.num_agents
-
-    # Generate a unique eval run folder
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"eval_pg_{num_agents}agents_{days_to_evaluate}days_{timestamp}"
-    output_folder = os.path.join("runs_with_battery", run_name)
-    logs_dir = os.path.join(output_folder, "logs")
-    plots_dir = os.path.join(output_folder, "plots")
-    for d in (logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Saving evaluation outputs to: {output_folder}")
-        
-    local_dim = env.observation_space.shape[1]
-    act_dim = env.action_space.shape[1]
-    
-    # Initialize PG agents
-    pg_agents = []
-    for i in range(num_agents):
-        agent = PGAgent(
-            state_dim=local_dim,
-            action_dim=act_dim,
-            lr=2e-4,
-            gamma=0.95,
-        )
-        
-        # Load individual agent model
-        agent_model_path = os.path.join(model_path, f"best_model_agent_{i}.pth")
-        if os.path.exists(agent_model_path):
-            agent.load(agent_model_path)
-            print(f"Loaded model for agent {i}")
-        else:
-            print(f"WARNING: Model file not found for agent {i}: {agent_model_path}")
-            # Alternative: try loading a single model for all agents
-            single_model_path = os.path.join(model_path, "best_model.pth")
-            if os.path.exists(single_model_path):
-                agent.load(single_model_path)
-                print(f"Loaded single model for agent {i}")
-        
-        agent.model.to(device).eval()
-        pg_agents.append(agent)
-
-    # Prepare logs
-    all_logs = []
-    daily_summaries = []
-    step_timing_list = []
-     
-    evaluation_start = time.time()
-
-    for day_idx in range(days_to_evaluate):
-        obs = env.reset()
-        done = False
-        step_count = 0
-        day_logs = []
-
-        while not done:
-            step_start_time = time.time()
-
-            # Select actions with PG
-            actions = []
-            with torch.no_grad():
-                for i in range(num_agents):
-                    # Convert observation to tensor and move to device
-                    state = torch.FloatTensor(obs[i]).unsqueeze(0).to(device)
-                    
-                    # Get action from actor network
-                    mean, log_std, _ = pg_agents[i].model(state)
-
-                    # For evaluation, use mean action (deterministic)
-                    action = mean.squeeze(0).cpu().numpy()
-                    
-                    # Clip to [0, 1] range
-                    action = np.clip(action, 0.0, 1.0)
-                    actions.append(action)
-            
-            actions = np.array(actions, dtype=np.float32)
-
-            next_obs, rewards, done, info = env.step(actions)
-             
-            # Consolidated Logging
-            step_end_time = time.time()
-            step_duration = step_end_time - step_start_time
-             
-            print(f"[Day {day_idx+1}, Step {step_count}] Step time: {step_duration:.6f} seconds")
-
-            step_timing_list.append({
-                "day": day_idx + 1,
-                "step": step_count,
-                "step_time_s": step_duration
-            })
-
-            grid_price_now = env.get_grid_price(step_count)
-            peer_price_now = info.get("peer_price", env.get_peer_price(step_count,
-                                float(info["p2p_sell"].sum()),
-                                float(info["p2p_buy"].sum())))
-
-            for i, hid in enumerate(house_ids):
-                is_battery_house = hid in env.batteries
-                p2p_buy = float(info["p2p_buy"][i])
-                p2p_sell = float(info["p2p_sell"][i])
-                charge_amount = float(info.get("charge_amount", [0]*num_agents)[i])
-                discharge_amount = float(info.get("discharge_amount", [0]*num_agents)[i])
-
-                day_logs.append({
-                    "day": day_idx + 1,
-                    "step": step_count,
-                    "house": hid,
-                    "grid_import_no_p2p": float(info["grid_import_no_p2p"][i]),
-                    "grid_import_with_p2p": float(info["grid_import_with_p2p"][i]),
-                    "grid_export": float(info.get("grid_export", [0]*num_agents)[i]),
-                    "p2p_buy": p2p_buy,
-                    "p2p_sell": p2p_sell,
-                    "actual_cost": float(info["costs"][i]),
-                    "baseline_cost": float(info["grid_import_no_p2p"][i]) * grid_price_now,
-                    "total_demand": float(env.demands[hid][step_count]),
-                    "total_solar": float(env.solars[hid][step_count]),
-                    "grid_price": grid_price_now,
-                    "peer_price": peer_price_now,
-                    "soc": (env.batteries[hid]["soc"] / env.batteries[hid]["max_capacity"]) if is_battery_house else np.nan,
-                    "degradation_cost": ((charge_amount + discharge_amount) * env.batteries[hid]["degradation_cost_per_kwh"]) if is_battery_house else 0.0,
-                    "reward": float(rewards[i]),
-                })
-
-            obs = next_obs
-            step_count += 1
-            if step_count >= eval_steps:
-                break
-         
-        day_df = pd.DataFrame(day_logs)
-        all_logs.extend(day_logs)
-
-        # Consolidated daily summary calculation
-        grouped_house = day_df.groupby("house").sum(numeric_only=True)
-        grouped_step = day_df.groupby("step").sum(numeric_only=True)
-
-        total_demand = grouped_step["total_demand"].sum()
-        total_solar = grouped_step["total_solar"].sum()
-        total_p2p_buy = grouped_house["p2p_buy"].sum()
-        total_p2p_sell = grouped_house["p2p_sell"].sum()
-
-        baseline_cost_per_house = grouped_house["baseline_cost"]
-        actual_cost_per_house = grouped_house["actual_cost"]
-        cost_savings_per_house = baseline_cost_per_house - actual_cost_per_house
-        day_total_cost_savings = cost_savings_per_house.sum()
-         
-        if baseline_cost_per_house.sum() > 0:
-            overall_cost_savings_pct = day_total_cost_savings / baseline_cost_per_house.sum()
-        else:
-            overall_cost_savings_pct = 0.0
-
-        baseline_import_per_house = grouped_house["grid_import_no_p2p"]
-        actual_import_per_house = grouped_house["grid_import_with_p2p"]
-        import_reduction_per_house = baseline_import_per_house - actual_import_per_house
-        day_total_import_reduction = import_reduction_per_house.sum()
-         
-        if baseline_import_per_house.sum() > 0:
-            overall_import_reduction_pct = day_total_import_reduction / baseline_import_per_house.sum()
-        else:
-            overall_import_reduction_pct = 0.0
-
-        fairness_cost_savings = compute_jains_fairness(cost_savings_per_house.values)
-        fairness_import_reduction = compute_jains_fairness(import_reduction_per_house.values)
-        fairness_rewards = compute_jains_fairness(grouped_house["reward"].values)
-        fairness_p2p_buy = compute_jains_fairness(grouped_house["p2p_buy"].values)
-        fairness_p2p_sell = compute_jains_fairness(grouped_house["p2p_sell"].values)
-        fairness_p2p_total = compute_jains_fairness((grouped_house["p2p_buy"] + grouped_house["p2p_sell"]).values)
-        day_total_degradation_cost = grouped_house["degradation_cost"].sum()
-
-        daily_summaries.append({
-            "day": day_idx + 1,
-            "day_total_demand": total_demand,
-            "day_total_solar": total_solar,
-            "day_p2p_buy": total_p2p_buy,
-            "day_p2p_sell": total_p2p_sell,
-            "cost_savings_abs": day_total_cost_savings,
-            "cost_savings_pct": overall_cost_savings_pct,
-            "fairness_cost_savings": fairness_cost_savings,
-            "grid_reduction_abs": day_total_import_reduction,
-            "grid_reduction_pct": overall_import_reduction_pct,
-            "fairness_grid_reduction": fairness_import_reduction,
-            "fairness_reward": fairness_rewards,
-            "fairness_p2p_buy": fairness_p2p_buy,
-            "fairness_p2p_sell": fairness_p2p_sell,
-            "fairness_p2p_total": fairness_p2p_total,
-            "total_degradation_cost": day_total_degradation_cost
-        })
-
-    # Final processing and saving
-    evaluation_end = time.time()
-    total_eval_time = evaluation_end - evaluation_start
-    print(f"\nEvaluation loop finished. Total time: {total_eval_time:.2f} seconds.")
-    print(f"Device used: {device}")
-
-    all_days_df = pd.DataFrame(all_logs)
-    combined_csv_path = os.path.join(logs_dir, "step_logs_all_days.csv")
-    all_days_df.to_csv(combined_csv_path, index=False)
-    print(f"Saved combined step-level logs to: {combined_csv_path}")
-
-    step_timing_df = pd.DataFrame(step_timing_list)
-    timing_csv_path = os.path.join(logs_dir, "step_timing_log.csv")
-    step_timing_df.to_csv(timing_csv_path, index=False)
-    print(f"Saved step timing logs to: {timing_csv_path}")
-
-    house_level_df = all_days_df.groupby("house").agg({
-        "baseline_cost": "sum",
-        "actual_cost": "sum",
-        "grid_import_no_p2p": "sum",
-        "grid_import_with_p2p": "sum",
-        "degradation_cost": "sum"
-    })
-    house_level_df["cost_savings"] = house_level_df["baseline_cost"] - house_level_df["actual_cost"]
-    house_level_df["import_reduction"] = house_level_df["grid_import_no_p2p"] - house_level_df["grid_import_with_p2p"]
-     
-    house_summary_csv = os.path.join(logs_dir, "summary_per_house.csv")
-    house_level_df.to_csv(house_summary_csv)
-    print(f"Saved final summary per house to: {house_summary_csv}")
-
-    fairness_grid_all = compute_jains_fairness(house_level_df["import_reduction"].values)
-    fairness_cost_all = compute_jains_fairness(house_level_df["cost_savings"].values)
-     
-    daily_summary_df = pd.DataFrame(daily_summaries)
-
-    total_cost_savings_all = daily_summary_df["cost_savings_abs"].sum()
-    total_baseline_cost_all = all_days_df.groupby('day')['baseline_cost'].sum().sum()
-    pct_cost_savings_all = total_cost_savings_all / total_baseline_cost_all if total_baseline_cost_all > 0 else 0.0
-
-    total_grid_reduction_all = daily_summary_df["grid_reduction_abs"].sum()
-    total_baseline_import_all = all_days_df.groupby('day')['grid_import_no_p2p'].sum().sum()
-    pct_grid_reduction_all = total_grid_reduction_all / total_baseline_import_all if total_baseline_import_all > 0 else 0.0
-
-    total_degradation_cost_all = daily_summary_df["total_degradation_cost"].sum()
-
-    # Calculate alternative performance metrics
-
-    # Grid Reduction During Solar Hours
-    agg_solar_per_step = all_days_df.groupby(['day', 'step'])['total_solar'].sum()
-    sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents)
-    sunny_df = all_days_df.set_index(['day', 'step'])[sunny_steps_mask].reset_index()
-    baseline_import_sunny = sunny_df['grid_import_no_p2p'].sum()
-    actual_import_sunny = sunny_df['grid_import_with_p2p'].sum()
-    grid_reduction_sunny_pct = 0.0
-    if baseline_import_sunny > 0:
-        grid_reduction_sunny_pct = (baseline_import_sunny - actual_import_sunny) / baseline_import_sunny
-
-    # Community Sourcing Rate
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    total_procured_energy = total_p2p_buy + total_actual_grid_import
-    community_sourcing_rate_pct = 0.0
-    if total_procured_energy > 0:
-        community_sourcing_rate_pct = total_p2p_buy / total_procured_energy
-
-    # Solar Sharing Efficiency
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    total_excess_solar = total_p2p_sell + total_grid_export
-    solar_sharing_efficiency_pct = 0.0
-    if total_excess_solar > 0:
-        solar_sharing_efficiency_pct = total_p2p_sell / total_excess_solar
-
-    baseline_cost_sunny = sunny_df['baseline_cost'].sum()
-    actual_cost_sunny = sunny_df['actual_cost'].sum()
-    cost_savings_sunny_pct = (baseline_cost_sunny - actual_cost_sunny) / baseline_cost_sunny if baseline_cost_sunny > 0 else 0.0
-
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    community_sourcing_rate_pct = total_p2p_buy / (total_p2p_buy + total_actual_grid_import) if (total_p2p_buy + total_actual_grid_import) > 0 else 0.0
-
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    solar_sharing_efficiency_pct = total_p2p_sell / (total_p2p_sell + total_grid_export) if (total_p2p_sell + total_grid_export) > 0 else 0.0
-
-    final_row = {
-        "day": "ALL_DAYS_SUMMARY",
-        "cost_savings_abs": total_cost_savings_all,
-        "cost_savings_pct": pct_cost_savings_all,
-        "grid_reduction_abs": total_grid_reduction_all,
-        "grid_reduction_pct": pct_grid_reduction_all,
-        "fairness_cost_savings": fairness_cost_all,
-        "fairness_grid_reduction": fairness_grid_all,
-        "total_degradation_cost": total_degradation_cost_all,
-        "grid_reduction_sunny_hours_pct": grid_reduction_sunny_pct,
-        "community_sourcing_rate_pct": community_sourcing_rate_pct,
-        "solar_sharing_efficiency_pct": solar_sharing_efficiency_pct,
-    }
-     
-    for col in daily_summary_df.columns:
-        if col not in final_row:
-            final_row[col] = np.nan
-    final_row_df = pd.DataFrame([final_row])
-
-    daily_summary_df = pd.concat([daily_summary_df, final_row_df], ignore_index=True)
-    summary_csv = os.path.join(logs_dir, "summary_per_day.csv")
-    daily_summary_df.to_csv(summary_csv, index=False)
-    print(f"Saved day-level summary with final multi-day row to: {summary_csv}")
-
-    # Final summary printout
-    print("\n================== EVALUATION SUMMARY ==================")
-    print(f"Evaluation finished for {days_to_evaluate} days.\n")
-     
-    print("--- Standard Metrics (24-Hour Average) ---")
-    print(f"Total grid reduction: {total_grid_reduction_all:.2f} kWh ({pct_grid_reduction_all:.2%})")
-    print(f"Total cost savings: ${total_cost_savings_all:.2f} ({pct_cost_savings_all:.2%})")
-    print(f"Jain's fairness on grid reduction: {fairness_grid_all:.3f}")
-    print(f"Jain's fairness on cost savings:   {fairness_cost_all:.3f}\n")
-
-    print("--- Alternative Metrics (Highlighting Peak Performance) ---")
-    print(f"Grid reduction during solar hours: {grid_reduction_sunny_pct:.2%}")
-    print(f"Cost savings during solar hours:   {cost_savings_sunny_pct:.2%}") 
-    print(f"Community sourcing rate:           {community_sourcing_rate_pct:.2%}")
-    print(f"Solar sharing efficiency:          {solar_sharing_efficiency_pct:.2%}")
-     
-    print("=========================================================")
-
-    # Plots
-    plot_daily_df = daily_summary_df[daily_summary_df["day"] != "ALL_DAYS_SUMMARY"].copy()
-    plot_daily_df["day"] = plot_daily_df["day"].astype(int)
-
-    # Daily Cost Savings Percentage
-    plt.figure(figsize=(12, 6))
-    plt.bar(plot_daily_df["day"], plot_daily_df["cost_savings_pct"] * 100, color='skyblue')
-    plt.xlabel("Day")
-    plt.ylabel("Cost Savings (%)")
-    plt.title("Daily Community Cost Savings Percentage")
-    plt.xticks(plot_daily_df["day"])
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_cost_savings_percentage.png"))
-    plt.close()
-
-    # Daily Total Demand vs. Solar
-    plt.figure(figsize=(12, 6))
-    bar_width = 0.4
-    days = plot_daily_df["day"]
-    plt.bar(days - bar_width/2, plot_daily_df["day_total_demand"], width=bar_width, label="Total Demand", color='coral')
-    plt.bar(days + bar_width/2, plot_daily_df["day_total_solar"], width=bar_width, label="Total Solar Generation", color='gold')
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Total Community Demand vs. Solar Generation Per Day")
-    plt.xticks(days)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_demand_vs_solar.png"))
-    plt.close()
-
-    # Combined Time Series of Energy Flows
-    step_group = all_days_df.groupby(["day", "step"]).sum(numeric_only=True).reset_index()
-    step_group["global_step"] = (step_group["day"] - 1) * env.num_steps + step_group["step"]
-     
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), sharex=True)
-     
-    # Subplot 1: Grid Import vs P2P Buy
-    ax1.plot(step_group["global_step"], step_group["grid_import_with_p2p"], label="Grid Import (with P2P)", color='r')
-    ax1.plot(step_group["global_step"], step_group["p2p_buy"], label="P2P Buy", color='g')
-    ax1.set_ylabel("Energy (kWh)")
-    ax1.set_title("Community Energy Consumption: Grid Import vs. P2P Buy")
-    ax1.legend()
-    ax1.grid(True, linestyle='--', alpha=0.6)
-
-    # Subplot 2: Grid Export vs P2P Sell
-    ax2.plot(step_group["global_step"], step_group["grid_export"], label="Grid Export", color='orange')
-    ax2.plot(step_group["global_step"], step_group["p2p_sell"], label="P2P Sell", color='b')
-    ax2.set_xlabel("Global Timestep")
-    ax2.set_ylabel("Energy (kWh)")
-    ax2.set_title("Community Energy Generation: Grid Export vs. P2P Sell")
-    ax2.legend()
-    ax2.grid(True, linestyle='--', alpha=0.6)
-     
-    plt.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "combined_energy_flows_timeseries.png"))
-    plt.close()
-
-    # Stacked Bar of Daily Energy Sources
-    daily_agg = all_days_df.groupby("day").sum(numeric_only=True)
-     
-    plt.figure(figsize=(12, 7))
-    plt.bar(daily_agg.index, daily_agg["grid_import_with_p2p"], label="Grid Import (with P2P)", color='crimson')
-    plt.bar(daily_agg.index, daily_agg["p2p_buy"], bottom=daily_agg["grid_import_with_p2p"], label="P2P Buy", color='limegreen')
-    plt.plot(daily_agg.index, daily_agg["grid_import_no_p2p"], label="Baseline Grid Import (No P2P)", color='blue', linestyle='--', marker='o')
-     
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Daily Energy Procurement: Baseline vs. P2P+Grid")
-    plt.xticks(daily_agg.index)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_energy_procurement_stacked.png"))
-    plt.close()
-
-    # Fairness Metrics Over Time
-    plt.figure(figsize=(12, 6))
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_cost_savings"], label="Cost Savings Fairness", marker='o')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_grid_reduction"], label="Grid Reduction Fairness", marker='s')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_reward"], label="Reward Fairness", marker='^')
-    plt.xlabel("Day")
-    plt.ylabel("Jain's Fairness Index")
-    plt.title("Daily Fairness Metrics")
-    plt.xticks(plot_daily_df["day"])
-    plt.ylim(0, 1.05)
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_fairness_metrics.png"))
-    plt.close()
-
-    # Per-House Savings and Reductions
-    fig, ax1 = plt.subplots(figsize=(15, 7))
-     
-    house_ids_str = house_level_df.index.astype(str)
-    bar_width = 0.4
-    index = np.arange(len(house_ids_str))
-
-    color1 = 'tab:green'
-    ax1.set_xlabel('House ID')
-    ax1.set_ylabel('Total Cost Savings ($)', color=color1)
-    ax1.bar(index - bar_width/2, house_level_df["cost_savings"], bar_width, label='Cost Savings', color=color1)
-    ax1.tick_params(axis='y', labelcolor=color1)
-    ax1.set_xticks(index)
-    ax1.set_xticklabels(house_ids_str, rotation=45, ha="right")
-     
-    ax2 = ax1.twinx()
-    color2 = 'tab:blue'
-    ax2.set_ylabel('Total Grid Import Reduction (kWh)', color=color2)
-    ax2.bar(index + bar_width/2, house_level_df["import_reduction"], bar_width, label='Import Reduction', color=color2)
-    ax2.tick_params(axis='y', labelcolor=color2)
-
-    plt.title(f'Total Cost Savings & Grid Import Reduction Per House (over {days_to_evaluate} days)')
-    fig.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "per_house_summary.png"))
-    plt.close()
-     
-    # Price Dynamics for a Single Day
-    day1_prices = all_days_df[all_days_df['day'] == 1][['step', 'grid_price', 'peer_price']].drop_duplicates()
-    plt.figure(figsize=(12, 6))
-    plt.plot(day1_prices['step'], day1_prices['grid_price'], label='Grid Price', color='darkorange')
-    plt.plot(day1_prices['step'], day1_prices['peer_price'], label='P2P Price', color='teal')
-    plt.xlabel("Timestep of Day")
-    plt.ylabel("Price ($/kWh)")
-    plt.title("Price Dynamics on Day 1")
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.6)
-    plt.savefig(os.path.join(plots_dir, "price_dynamics_day1.png"))
-    plt.close()
-     
-    # Battery State of Charge for Sample Houses
-    day1_df = all_days_df[all_days_df['day'] == 1]
-    battery_houses = day1_df.dropna(subset=['soc'])['house'].unique()
-     
-    if len(battery_houses) > 0:
-        sample_houses = battery_houses[:min(4, len(battery_houses))]
-        plt.figure(figsize=(12, 6))
-        for house in sample_houses:
-            house_df = day1_df[day1_df['house'] == house]
-            plt.plot(house_df['step'], house_df['soc'] * 100, label=f'House {house}')
-         
-        plt.xlabel("Timestep of Day")
-        plt.ylabel("State of Charge (%)")
-        plt.title("Battery SoC on Day 1 for Sample Houses")
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.6)
-        plt.savefig(os.path.join(plots_dir, "soc_dynamics_day1.png"))
-        plt.close()
-
-    print("All plots have been generated and saved. Evaluation complete.")
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/PG/pg_train.py

@@ -1,373 +0,0 @@
-import os
-import sys
-import re
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-import pandas as pd
-import time
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from PG.trainer.pg import PGAgent
- 
-def main():
-    STATE_TO_RUN = "pennsylvania"  # "pennsylvania" or "colorado" or "oklahoma"
-
-    # Set the path to your training data
-    DATA_FILE_PATH = "/path/to/project/training/5houses_152days_TRAIN.csv"
-    num_episodes = 10000
-    batch_size = 256    
-    checkpoint_interval = 100000
-    window_size = 32
-
-    env = SolarSys(
-        data_path=DATA_FILE_PATH,
-        state=STATE_TO_RUN,
-        time_freq="15T"
-    )
-
-    # Sanity check: env I/O shapes
-    print("Observation space:", env.observation_space)
-    print("Action space     :", env.action_space)
-
-    # Reset and inspect obs
-    obs = env.reset()
-    print(f"Reset returned {len(obs)} agent observations; each obs shape: {np.array(obs).shape}")
-
-    # Sample random actions and do one step
-    dummy_actions = np.random.rand(env.num_agents, env.action_space.shape[1]).astype(np.float32)
-    next_obs, rewards, done, info = env.step(dummy_actions)
-    print(f"Step outputs → next_obs: {len(next_obs)}×{np.array(next_obs).shape[1]}, "
-          f"rewards: {len(rewards)}, done: {done}")
-    print("Info keys:", list(info.keys()))
-
-    # Count the number of houses in each group
-    env.group_counts = {
-        0: env.agent_groups.count(0),
-        1: env.agent_groups.count(1)
-    }
-    print(f"Number of houses in each group: {env.group_counts}")
-
-    max_steps = env.num_steps
-
-    # Dims from the env
-    num_agents = env.num_agents
-    local_state_dim = env.observation_space.shape[1]
-    action_dim = env.action_space.shape[1]
-
-    # Build a unique run directory
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"pg_{STATE_TO_RUN}_{num_agents}agents_{num_episodes}eps_{timestamp}"
-    root_dir = os.path.join("FINALE_FINALE_FINALE", run_name)
-    os.makedirs(root_dir, exist_ok=True)
-    print(f"Saving training outputs to: {root_dir}")
-
-    logs_dir = os.path.join(root_dir, "logs")
-    plots_dir = os.path.join(root_dir, "plots")
-    os.makedirs(logs_dir, exist_ok=True)
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Create PG agents with use_baseline parameter
-    pg_agents = [
-        PGAgent(
-            state_dim=local_state_dim, 
-            action_dim=action_dim, 
-            lr=2e-4,
-            gamma=0.95,
-            critic_loss_coef=0.5
-        )
-        for _ in range(num_agents)
-    ]
-
-    # Tracking / Logging Variables
-    episode_rewards = []             
-    episode_total_rewards = []       
-    block_mean_rewards = []          
-    block_total_rewards = []         
-
-    agent_rewards_log = [[] for _ in range(num_agents)]
-    best_mean_reward = -1e9
-    best_model_path = os.path.join(logs_dir, "best_model.pth")
-
-    daily_rewards = []               
-    monthly_rewards = []             
-
-    training_start_time = time.time()
-    episode_durations = []
-    total_steps_global = 0
-    episode_log_data = []
-    performance_metrics_log = []
-
-    agent_charge_log = [[] for _ in range(num_agents)]  
-    agent_discharge_log = [[] for _ in range(num_agents)]
-
-    # Training Loop
-    for episode in range(1, num_episodes + 1):
-        episode_start_time = time.time()
-
-        obs = np.array(env.reset(), dtype=np.float32)
-
-        if episode > 1:
-            last_episode_metrics = env.get_episode_metrics()
-            last_episode_metrics['Episode'] = episode - 1
-            performance_metrics_log.append(last_episode_metrics)
-
-        total_reward = np.zeros(num_agents, dtype=np.float32)
-        done = False
-        step_count = 0
-        day_logs = []
-        episode_charges = [[] for _ in range(num_agents)]
-        episode_discharges = [[] for _ in range(num_agents)]
-
-        # Main training loop for a single episode
-        while not done:
-            # Action Selection: Each PG agent acts independently
-            actions = []
-            for i, agent in enumerate(pg_agents):
-                agent_action = agent.select_action(obs[i])
-                actions.append(agent_action)
-            actions = np.array(actions, dtype=np.float32)
-
-            # Step the environment
-            next_obs_list, rewards, done, info = env.step(actions)
-            next_obs = np.array(next_obs_list, dtype=np.float32)
-
-            # Store Rewards: Each agent stores its own reward
-            for i, agent in enumerate(pg_agents):
-                agent.rewards.append(rewards[i])
-                agent.dones.append(done)
-
-            total_reward += rewards
-            obs = next_obs
-            step_count += 1
-            total_steps_global += 1
-
-            day_logs.append({
-                "step": step_count - 1,
-                "grid_import_no_p2p": info["grid_import_no_p2p"],
-                "grid_import_with_p2p": info["grid_import_with_p2p"],
-                "p2p_buy": info["p2p_buy"],
-                "p2p_sell": info["p2p_sell"],
-                "costs": info["costs"],
-                "charge_amount": info.get("charge_amount", np.zeros(num_agents)),
-                "discharge_amount": info.get("discharge_amount", np.zeros(num_agents))
-            })
-
-            # Track actual charge/discharge actions from the environment
-            for i in range(num_agents):
-                episode_charges[i].append(info["charge_amount"][i])
-                episode_discharges[i].append(info["discharge_amount"][i])
-
-            if step_count >= max_steps:
-                break
-
-        # After each episode
-        sum_ep_reward = float(np.sum(total_reward))
-        mean_ep_reward = float(np.mean(total_reward))
-
-        episode_total_rewards.append(sum_ep_reward)
-        episode_rewards.append(mean_ep_reward)
-        daily_rewards.append(mean_ep_reward)
-
-        if len(daily_rewards) % window_size == 0:
-            last_totals = episode_total_rewards[-window_size:]
-            block_sum = sum(last_totals)
-            block_total_rewards.append(block_sum)
-
-            last_means = daily_rewards[-window_size:]
-            block_mean = sum(last_means) / window_size
-            block_mean_rewards.append(block_mean)
-
-            block_idx = len(block_mean_rewards)
-            print(
-                f"→ Completed Block {block_idx} "
-                f"| Episodes {(block_idx - 1) * window_size + 1}–{block_idx * window_size} "
-                f"| Block Total Reward: {block_sum:.3f} "
-                f"| Block Mean Reward: {block_mean:.3f}"
-            )
-
-        for i in range(num_agents):
-            agent_rewards_log[i].append(total_reward[i])
-            agent_charge_log[i].append(np.mean(episode_charges[i]))
-            agent_discharge_log[i].append(np.mean(episode_discharges[i]))
-
-        steps_data = []
-        for entry in day_logs:
-            steps_data.append({
-                "step": entry["step"],
-                "p2p_buy_sum":  float(np.sum(entry["p2p_buy"])),
-                "p2p_sell_sum": float(np.sum(entry["p2p_sell"])),
-                "grid_import_no_p2p_sum":   float(np.sum(entry["grid_import_no_p2p"])),
-                "grid_import_with_p2p_sum": float(np.sum(entry["grid_import_with_p2p"]))
-            })
-
-        baseline_cost = np.sum([np.sum(entry["grid_import_no_p2p"]) * env.get_grid_price(entry["step"])
-                                for entry in day_logs])
-        actual_cost = np.sum([np.sum(entry["costs"]) for entry in day_logs])
-        cost_reduction = (baseline_cost - actual_cost) / (baseline_cost + 1e-8)
-
-        # UPDATE STEP: Update each PG agent independently
-        for agent in pg_agents:
-            agent.update()
-
-        # Save best models
-        if mean_ep_reward > best_mean_reward:
-            best_mean_reward = mean_ep_reward
-            for i, agent in enumerate(pg_agents):
-                agent_path = os.path.join(logs_dir, f"best_model_agent_{i}.pth")
-                agent.save(agent_path)
-
-        if episode % checkpoint_interval == 0:
-            for i, agent in enumerate(pg_agents):
-                ckpt_path = os.path.join(logs_dir, f"checkpoint_{episode}_agent_{i}.pth")
-                agent.save(ckpt_path)
-
-        episode_end_time = time.time()
-        episode_duration = episode_end_time - episode_start_time
-
-        print(
-            f"Episode {episode}/{num_episodes} "
-            f"| Time per Episode: {episode_duration:.2f}s "
-            f"| Steps: {step_count} "
-            f"| Mean Reward: {mean_ep_reward:.3f} "
-            f"| Cost Reduction: {cost_reduction:.2%}"
-        )
-
-        episode_log_data.append({
-            "Episode": episode,
-            "Steps": step_count,
-            "Mean_Reward": mean_ep_reward,
-            "Total_Reward": sum_ep_reward,
-            "Cost_Reduction_Pct": cost_reduction * 100,
-            "Baseline_Cost": baseline_cost,
-            "Actual_Cost": actual_cost,
-            "Episode_Duration": episode_duration,
-            "Total_Charge": np.sum([np.sum(entry["charge_amount"]) for entry in day_logs]),
-            "Total_Discharge": np.sum([np.sum(entry["discharge_amount"]) for entry in day_logs])
-        })
-
-        # Periodic performance logging
-        if episode % 100 == 0:
-            avg_reward_last_100 = np.mean(daily_rewards[-100:]) if len(daily_rewards) >= 100 else np.mean(daily_rewards)
-            print(f"  → Average reward (last 100 episodes): {avg_reward_last_100:.3f}")
-
-    # Final episode metrics
-    final_episode_metrics = env.get_episode_metrics()
-    final_episode_metrics['Episode'] = num_episodes
-    performance_metrics_log.append(final_episode_metrics)
-
-    training_end_time = time.time()
-    total_training_time = training_end_time - training_start_time
-
-    # Save final models
-    print("\nSaving final models...")
-    for i, agent in enumerate(pg_agents):
-        final_path = os.path.join(logs_dir, f"final_model_agent_{i}.pth")
-        agent.save(final_path)
-
-    np.save(os.path.join(logs_dir, "agent_rewards.npy"), np.array(agent_rewards_log))
-    np.save(os.path.join(logs_dir, "mean_rewards.npy"),  np.array(episode_rewards))
-    np.save(os.path.join(logs_dir, "total_rewards.npy"), np.array(episode_total_rewards))
-
-    # Create DataFrames
-    df_rewards_log = pd.DataFrame(episode_log_data)
-    df_perf_log = pd.DataFrame(performance_metrics_log)
-    df_final_log = pd.merge(df_rewards_log, df_perf_log.drop(columns=[
-        'degradation_cost_over_time',
-        'cost_savings_over_time',
-        'grid_reduction_over_time'
-    ]), on="Episode")
-
-    # Helper: centered moving average
-    def moving_avg(series, window):
-        return pd.Series(series).rolling(window=window, center=True, min_periods=1).mean().to_numpy()
-
-    ma_window = 300
-    episodes = np.arange(1, num_episodes + 1)
-
-    # Mean Reward moving average
-    reward_ma = moving_avg(df_final_log["Mean_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, reward_ma, linewidth=2, label=f"Mean Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Mean Reward")
-    plt.title("PG: Mean Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "mean_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Total Reward moving average
-    total_ma = moving_avg(df_final_log["Total_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, total_ma, linewidth=2, label=f"Total Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Total Reward")
-    plt.title("PG: Total Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "total_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Cost Reduction (%) moving average
-    cost_ma = moving_avg(df_final_log["Cost_Reduction_Pct"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, cost_ma, linewidth=2, label="Cost Reduction MA (%)")
-    plt.xlabel("Episode")
-    plt.ylabel("Cost Reduction (%)")
-    plt.title("PG: Cost Reduction Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "cost_reduction_ma.png"), dpi=200)
-    plt.close()
-
-    # Battery Degradation Cost moving average
-    degradation_ma = moving_avg(df_final_log["battery_degradation_cost_total"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, degradation_ma, linewidth=2, label=f"Degradation Cost MA (win={ma_window})", color='purple')
-    plt.xlabel("Episode")
-    plt.ylabel("Total Degradation Cost ($)")
-    plt.title("PG: Battery Degradation Cost Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "degradation_cost_ma.png"), dpi=200)
-    plt.close()
-
-    print(f"\nAll moving-average plots saved to: {plots_dir}")
-
-    # Save Final Logs to CSV
-    total_time_row = pd.DataFrame([{
-        "Episode": "Total_Training_Time",
-        "Episode_Duration": total_training_time
-    }])
-    df_to_save = pd.concat([df_final_log, total_time_row], ignore_index=True)
-
-    log_csv_path = os.path.join(logs_dir, "training_performance_log.csv")
-
-    columns_to_save = [
-        "Episode",
-        "Mean_Reward",
-        "Total_Reward",
-        "Cost_Reduction_Pct",
-        "Episode_Duration",
-        "battery_degradation_cost_total",
-    ]
-    df_to_save = df_to_save[columns_to_save]
-
-    df_to_save.to_csv(log_csv_path, index=False)
-
-    print(f"Saved comprehensive training performance log to: {log_csv_path}")
-
-    # Final Timings Printout
-    print("\n" + "="*50)
-    print("TRAINING COMPLETE".center(50))
-    print(f"Total training time: {total_training_time:.2f} seconds")
-    print(f"Device used: {pg_agents[0].device}")
-    print("="*50)
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/PG/trainer/pg.py

@@ -1,96 +0,0 @@
-import torch
-import torch.nn as nn
-from torch.distributions import Normal
-import numpy as np
-
-class SharedActorCritic(nn.Module):
-    def __init__(self, state_dim, action_dim):
-        super(SharedActorCritic, self).__init__()
-        self.feature_extractor = nn.Sequential(
-            nn.Linear(state_dim, 128),
-            nn.ReLU(),
-            nn.Linear(128, 128),
-            nn.ReLU()
-        )
-        self.actor_head = nn.Linear(128, action_dim * 2)
-        self.critic_head = nn.Linear(128, 1)
-
-    def forward(self, state):
-        features = self.feature_extractor(state)
-        action_params = self.actor_head(features)
-        mean, log_std = torch.chunk(action_params, 2, dim=-1)
-        value = self.critic_head(features)
-        return mean, log_std, value
-
-class PGAgent:
-    def __init__(self, state_dim, action_dim, lr=3e-4, gamma=0.95, gae_lambda=0.95, critic_loss_coef=0.5):
-        self.gamma = gamma
-        self.gae_lambda = gae_lambda
-        self.critic_loss_coef = critic_loss_coef
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.model = SharedActorCritic(state_dim, action_dim).to(self.device)
-        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)
-        self.log_probs = []
-        self.rewards = []
-        self.values = []
-        self.dones = []
-        self.log_std_min = -20
-        self.log_std_max = 2
-
-    def select_action(self, state):
-        state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
-        mean, log_std, value = self.model(state_tensor)
-        log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
-        std = torch.exp(log_std)
-        dist = Normal(mean, std)
-        action = dist.sample()
-        log_prob = dist.log_prob(action).sum(dim=-1)
-        self.log_probs.append(log_prob)
-        self.values.append(value)
-        return np.clip(action.squeeze(0).cpu().detach().numpy(), 0.0, 1.0)
-
-    def update(self):
-        if not self.rewards:
-            return
-        next_value = 0
-        values = torch.cat(self.values).squeeze().detach().cpu().numpy()
-        advantages, returns = self._calculate_gae_advantages(self.rewards, values, self.dones, next_value)
-        log_probs = torch.cat(self.log_probs)
-        advantages = torch.tensor(advantages, dtype=torch.float32, device=self.device)
-        returns = torch.tensor(returns, dtype=torch.float32, device=self.device)
-        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
-        actor_loss = -(log_probs * advantages).mean()
-        critic_values = torch.cat(self.values).squeeze()
-        critic_loss = nn.MSELoss()(critic_values, returns)
-        total_loss = actor_loss + self.critic_loss_coef * critic_loss
-        self.optimizer.zero_grad()
-        total_loss.backward()
-        torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5)
-        self.optimizer.step()
-        self.rewards = []
-        self.log_probs = []
-        self.values = []
-        self.dones = []
-
-    def _calculate_gae_advantages(self, rewards, values, dones, next_value):
-        advantages = np.zeros_like(rewards, dtype=np.float32)
-        last_advantage = 0
-        for t in reversed(range(len(rewards))):
-            mask = 1.0 - dones[t]
-            v_next = values[t + 1] if t < len(rewards) - 1 else next_value
-            delta = rewards[t] + self.gamma * v_next * mask - values[t]
-            last_advantage = delta + self.gamma * self.gae_lambda * last_advantage * mask
-            advantages[t] = last_advantage
-        returns = advantages + values
-        return advantages, returns
-
-    def save(self, path):
-        torch.save({
-            'model_state_dict': self.model.state_dict(),
-            'optimizer_state_dict': self.optimizer.state_dict(),
-        }, path)
-
-    def load(self, path):
-        checkpoint = torch.load(path, map_location=self.device)
-        self.model.load_state_dict(checkpoint['model_state_dict'])
-        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

Other_algorithms/Flat_System/maddpg/maddpg_evaluation.py

@@ -1,495 +0,0 @@
-# maddpg_evaluate.py
-import os
-import sys
-import time
-import re
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import torch
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from maddpg.trainer.maddpg import MADDPG
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-def compute_jains_fairness(values: np.ndarray) -> float:
-    if len(values) == 0:
-        return 0.0
-    if np.all(values == 0):
-        return 1.0
-    num = (values.sum())**2
-    den = len(values) * (values**2).sum()
-    return num / den
-
-def main():
-    # User parameters
-    MODEL_PATH = "/path/to/project/maddpg_para_sharing_oklahoma_5agents_10000eps/logs/best_model.pth"
-    DATA_PATH = "/path/to/project/testing/5houses_30days_TEST.csv"
-    DAYS_TO_EVALUATE = 30
-
-    model_path = MODEL_PATH
-    data_path = DATA_PATH
-    days_to_evaluate = DAYS_TO_EVALUATE
-    SOLAR_THRESHOLD = 0.4
-
-    state_match = re.search(r"maddpg_para_sharing_(oklahoma|colorado|pennsylvania)_", model_path)
-    if not state_match:
-        raise ValueError(
-            "Could not automatically detect the state (oklahoma, colorado, or pennsylvania) "
-            "from the model path. Please ensure your model's parent folder is named correctly, "
-            "e.g., 'maddpg_para_sharing_oklahoma_...'"
-        )
-    detected_state = state_match.group(1)
-    print(f"--- Detected state: {detected_state.upper()} ---")
-
-    # Env setup
-    env = SolarSys(
-        data_path=data_path,
-        state=detected_state,
-        time_freq="15T"
-    )
-    eval_steps = env.num_steps
-    house_ids = env.house_ids
-    num_agents = env.num_agents
-
-    # Generate a unique eval run folder
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"eval_maddpg_para_sharing_{num_agents}agents_{days_to_evaluate}days_{timestamp}"
-    output_folder = os.path.join("runs_with_battery", run_name)
-    logs_dir = os.path.join(output_folder, "logs")
-    plots_dir = os.path.join(output_folder, "plots")
-    for d in (logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Saving evaluation outputs to: {output_folder}")
-        
-    local_state_dim = env.observation_space.shape[1]
-    action_dim = env.action_space.shape[1]
-   
-    # Instantiate MADDPG agent
-    maddpg = MADDPG(
-        num_agents=num_agents,
-        state_dim=local_state_dim,
-        action_dim=action_dim
-    )
-
-    # Load MADDPG checkpoint
-    maddpg.load(model_path)
-    
-    maddpg.actor.eval()
-    maddpg.critic.eval()
-    maddpg.target_actor.eval()
-    maddpg.target_critic.eval()
-
-    # Prepare logs
-    all_logs = []
-    daily_summaries = []
-    step_timing_list = []
-     
-    evaluation_start = time.time()
-
-    for day_idx in range(days_to_evaluate):
-        obs = env.reset()
-        done = False
-        step_count = 0
-        day_logs = []
-
-        while not done:
-            step_start_time = time.time()
-            
-            # Select actions with MADDPG
-            actions = maddpg.select_actions(obs, evaluate=True)
-
-            next_obs, rewards, done, info = env.step(actions)
-            
-            # Consolidated Logging
-            step_end_time = time.time()
-            step_duration = step_end_time - step_start_time
-             
-            print(f"[Day {day_idx+1}, Step {step_count}] Step time: {step_duration:.6f} seconds")
-
-            step_timing_list.append({
-                "day": day_idx + 1,
-                "step": step_count,
-                "step_time_s": step_duration
-            })
-
-            grid_price_now = env.get_grid_price(step_count)
-            peer_price_now = info.get("peer_price", env.get_peer_price(step_count,
-                                float(info["p2p_sell"].sum()),
-                                float(info["p2p_buy"].sum())))
-
-            for i, hid in enumerate(house_ids):
-                is_battery_house = hid in env.batteries
-                p2p_buy = float(info["p2p_buy"][i])
-                p2p_sell = float(info["p2p_sell"][i])
-                charge_amount = float(info.get("charge_amount", [0]*num_agents)[i])
-                discharge_amount = float(info.get("discharge_amount", [0]*num_agents)[i])
-
-                day_logs.append({
-                    "day": day_idx + 1,
-                    "step": step_count,
-                    "house": hid,
-                    "grid_import_no_p2p": float(info["grid_import_no_p2p"][i]),
-                    "grid_import_with_p2p": float(info["grid_import_with_p2p"][i]),
-                    "grid_export": float(info.get("grid_export", [0]*num_agents)[i]),
-                    "p2p_buy": p2p_buy,
-                    "p2p_sell": p2p_sell,
-                    "actual_cost": float(info["costs"][i]),
-                    "baseline_cost": float(info["grid_import_no_p2p"][i]) * grid_price_now,
-                    "total_demand": float(env.demands[hid][step_count]),
-                    "total_solar": float(env.solars[hid][step_count]),
-                    "grid_price": grid_price_now,
-                    "peer_price": peer_price_now,
-                    "soc": (env.batteries[hid]["soc"] / env.batteries[hid]["max_capacity"]) if is_battery_house else np.nan,
-                    "degradation_cost": ((charge_amount + discharge_amount) * env.batteries[hid]["degradation_cost_per_kwh"]) if is_battery_house else 0.0,
-                    "reward": float(rewards[i]),
-                })
-
-            obs = next_obs
-            step_count += 1
-            if step_count >= eval_steps:
-                break
-         
-        day_df = pd.DataFrame(day_logs)
-        all_logs.extend(day_logs)
-
-        # Consolidated daily summary calculation
-        grouped_house = day_df.groupby("house").sum(numeric_only=True)
-        grouped_step = day_df.groupby("step").sum(numeric_only=True)
-
-        total_demand = grouped_step["total_demand"].sum()
-        total_solar = grouped_step["total_solar"].sum()
-        total_p2p_buy = grouped_house["p2p_buy"].sum()
-        total_p2p_sell = grouped_house["p2p_sell"].sum()
-
-        baseline_cost_per_house = grouped_house["baseline_cost"]
-        actual_cost_per_house = grouped_house["actual_cost"]
-        cost_savings_per_house = baseline_cost_per_house - actual_cost_per_house
-        day_total_cost_savings = cost_savings_per_house.sum()
-         
-        if baseline_cost_per_house.sum() > 0:
-            overall_cost_savings_pct = day_total_cost_savings / baseline_cost_per_house.sum()
-        else:
-            overall_cost_savings_pct = 0.0
-
-        baseline_import_per_house = grouped_house["grid_import_no_p2p"]
-        actual_import_per_house = grouped_house["grid_import_with_p2p"]
-        import_reduction_per_house = baseline_import_per_house - actual_import_per_house
-        day_total_import_reduction = import_reduction_per_house.sum()
-         
-        if baseline_import_per_house.sum() > 0:
-            overall_import_reduction_pct = day_total_import_reduction / baseline_import_per_house.sum()
-        else:
-            overall_import_reduction_pct = 0.0
-
-        fairness_cost_savings = compute_jains_fairness(cost_savings_per_house.values)
-        fairness_import_reduction = compute_jains_fairness(import_reduction_per_house.values)
-        fairness_rewards = compute_jains_fairness(grouped_house["reward"].values)
-        fairness_p2p_buy = compute_jains_fairness(grouped_house["p2p_buy"].values)
-        fairness_p2p_sell = compute_jains_fairness(grouped_house["p2p_sell"].values)
-        fairness_p2p_total = compute_jains_fairness((grouped_house["p2p_buy"] + grouped_house["p2p_sell"]).values)
-        day_total_degradation_cost = grouped_house["degradation_cost"].sum()
-
-        daily_summaries.append({
-            "day": day_idx + 1,
-            "day_total_demand": total_demand,
-            "day_total_solar": total_solar,
-            "day_p2p_buy": total_p2p_buy,
-            "day_p2p_sell": total_p2p_sell,
-            "cost_savings_abs": day_total_cost_savings,
-            "cost_savings_pct": overall_cost_savings_pct,
-            "fairness_cost_savings": fairness_cost_savings,
-            "grid_reduction_abs": day_total_import_reduction,
-            "grid_reduction_pct": overall_import_reduction_pct,
-            "fairness_grid_reduction": fairness_import_reduction,
-            "fairness_reward": fairness_rewards,
-            "fairness_p2p_buy": fairness_p2p_buy,
-            "fairness_p2p_sell": fairness_p2p_sell,
-            "fairness_p2p_total": fairness_p2p_total,
-            "total_degradation_cost": day_total_degradation_cost
-        })
-
-    # Final processing and saving
-    evaluation_end = time.time()
-    total_eval_time = evaluation_end - evaluation_start
-    print(f"\nEvaluation loop finished. Total time: {total_eval_time:.2f} seconds.")
-
-    all_days_df = pd.DataFrame(all_logs)
-    combined_csv_path = os.path.join(logs_dir, "step_logs_all_days.csv")
-    all_days_df.to_csv(combined_csv_path, index=False)
-    print(f"Saved combined step-level logs to: {combined_csv_path}")
-
-    step_timing_df = pd.DataFrame(step_timing_list)
-    timing_csv_path = os.path.join(logs_dir, "step_timing_log.csv")
-    step_timing_df.to_csv(timing_csv_path, index=False)
-    print(f"Saved step timing logs to: {timing_csv_path}")
-
-    house_level_df = all_days_df.groupby("house").agg({
-        "baseline_cost": "sum",
-        "actual_cost": "sum",
-        "grid_import_no_p2p": "sum",
-        "grid_import_with_p2p": "sum",
-        "degradation_cost": "sum"
-    })
-    house_level_df["cost_savings"] = house_level_df["baseline_cost"] - house_level_df["actual_cost"]
-    house_level_df["import_reduction"] = house_level_df["grid_import_no_p2p"] - house_level_df["grid_import_with_p2p"]
-     
-    house_summary_csv = os.path.join(logs_dir, "summary_per_house.csv")
-    house_level_df.to_csv(house_summary_csv)
-    print(f"Saved final summary per house to: {house_summary_csv}")
-
-    fairness_grid_all = compute_jains_fairness(house_level_df["import_reduction"].values)
-    fairness_cost_all = compute_jains_fairness(house_level_df["cost_savings"].values)
-     
-    daily_summary_df = pd.DataFrame(daily_summaries)
-
-    total_cost_savings_all = daily_summary_df["cost_savings_abs"].sum()
-    total_baseline_cost_all = all_days_df.groupby('day')['baseline_cost'].sum().sum()
-    pct_cost_savings_all = total_cost_savings_all / total_baseline_cost_all if total_baseline_cost_all > 0 else 0.0
-
-    total_grid_reduction_all = daily_summary_df["grid_reduction_abs"].sum()
-    total_baseline_import_all = all_days_df.groupby('day')['grid_import_no_p2p'].sum().sum()
-    pct_grid_reduction_all = total_grid_reduction_all / total_baseline_import_all if total_baseline_import_all > 0 else 0.0
-
-    total_degradation_cost_all = daily_summary_df["total_degradation_cost"].sum()
-
-    # Calculate alternative performance metrics
-
-    # Grid Reduction During Solar Hours
-    agg_solar_per_step = all_days_df.groupby(['day', 'step'])['total_solar'].sum()
-    sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents)
-    sunny_df = all_days_df.set_index(['day', 'step'])[sunny_steps_mask].reset_index()
-    baseline_import_sunny = sunny_df['grid_import_no_p2p'].sum()
-    actual_import_sunny = sunny_df['grid_import_with_p2p'].sum()
-    grid_reduction_sunny_pct = 0.0
-    if baseline_import_sunny > 0:
-        grid_reduction_sunny_pct = (baseline_import_sunny - actual_import_sunny) / baseline_import_sunny
-
-    # Community Sourcing Rate
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    total_procured_energy = total_p2p_buy + total_actual_grid_import
-    community_sourcing_rate_pct = 0.0
-    if total_procured_energy > 0:
-        community_sourcing_rate_pct = total_p2p_buy / total_procured_energy
-
-    # Solar Sharing Efficiency
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    total_excess_solar = total_p2p_sell + total_grid_export
-    solar_sharing_efficiency_pct = 0.0
-    if total_excess_solar > 0:
-        solar_sharing_efficiency_pct = total_p2p_sell / total_excess_solar
-
-    # Cost savings in sunny hours
-    baseline_cost_sunny = sunny_df['baseline_cost'].sum()
-    actual_cost_sunny = sunny_df['actual_cost'].sum()
-    cost_savings_sunny_pct = (baseline_cost_sunny - actual_cost_sunny) / baseline_cost_sunny if baseline_cost_sunny > 0 else 0.0
-
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    community_sourcing_rate_pct = total_p2p_buy / (total_p2p_buy + total_actual_grid_import) if (total_p2p_buy + total_actual_grid_import) > 0 else 0.0
-
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    solar_sharing_efficiency_pct = total_p2p_sell / (total_p2p_sell + total_grid_export) if (total_p2p_sell + total_grid_export) > 0 else 0.0
-
-    final_row = {
-        "day": "ALL_DAYS_SUMMARY",
-        "cost_savings_abs": total_cost_savings_all,
-        "cost_savings_pct": pct_cost_savings_all,
-        "grid_reduction_abs": total_grid_reduction_all,
-        "grid_reduction_pct": pct_grid_reduction_all,
-        "fairness_cost_savings": fairness_cost_all,
-        "fairness_grid_reduction": fairness_grid_all,
-        "total_degradation_cost": total_degradation_cost_all,
-        "grid_reduction_sunny_hours_pct": grid_reduction_sunny_pct,
-        "community_sourcing_rate_pct": community_sourcing_rate_pct,
-        "solar_sharing_efficiency_pct": solar_sharing_efficiency_pct,
-    }
-     
-    for col in daily_summary_df.columns:
-        if col not in final_row:
-            final_row[col] = np.nan
-    final_row_df = pd.DataFrame([final_row])
-
-    daily_summary_df = pd.concat([daily_summary_df, final_row_df], ignore_index=True)
-    summary_csv = os.path.join(logs_dir, "summary_per_day.csv")
-    daily_summary_df.to_csv(summary_csv, index=False)
-    print(f"Saved day-level summary with final multi-day row to: {summary_csv}")
-
-    # Final summary printout
-    print("\n================== EVALUATION SUMMARY ==================")
-    print(f"Evaluation finished for {days_to_evaluate} days.\n")
-     
-    print("--- Standard Metrics (24-Hour Average) ---")
-    print(f"Total grid reduction: {total_grid_reduction_all:.2f} kWh ({pct_grid_reduction_all:.2%})")
-    print(f"Total cost savings: ${total_cost_savings_all:.2f} ({pct_cost_savings_all:.2%})")
-    print(f"Jain's fairness on grid reduction: {fairness_grid_all:.3f}")
-    print(f"Jain's fairness on cost savings:   {fairness_cost_all:.3f}\n")
-
-    print("--- Alternative Metrics (Highlighting Peak Performance) ---")
-    print(f"Grid reduction during solar hours: {grid_reduction_sunny_pct:.2%}")
-    print(f"Cost savings during solar hours:   {cost_savings_sunny_pct:.2%}") 
-    print(f"Community sourcing rate:           {community_sourcing_rate_pct:.2%}")
-    print(f"Solar sharing efficiency:          {solar_sharing_efficiency_pct:.2%}")
-     
-    print("=========================================================")
-
-    # Plots
-    plot_daily_df = daily_summary_df[daily_summary_df["day"] != "ALL_DAYS_SUMMARY"].copy()
-    plot_daily_df["day"] = plot_daily_df["day"].astype(int)
-
-    # Daily Cost Savings Percentage
-    plt.figure(figsize=(12, 6))
-    plt.bar(plot_daily_df["day"], plot_daily_df["cost_savings_pct"] * 100, color='skyblue')
-    plt.xlabel("Day")
-    plt.ylabel("Cost Savings (%)")
-    plt.title("Daily Community Cost Savings Percentage")
-    plt.xticks(plot_daily_df["day"])
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_cost_savings_percentage.png"))
-    plt.close()
-
-    # Daily Total Demand vs. Solar
-    plt.figure(figsize=(12, 6))
-    bar_width = 0.4
-    days = plot_daily_df["day"]
-    plt.bar(days - bar_width/2, plot_daily_df["day_total_demand"], width=bar_width, label="Total Demand", color='coral')
-    plt.bar(days + bar_width/2, plot_daily_df["day_total_solar"], width=bar_width, label="Total Solar Generation", color='gold')
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Total Community Demand vs. Solar Generation Per Day")
-    plt.xticks(days)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_demand_vs_solar.png"))
-    plt.close()
-
-    # Combined Time Series of Energy Flows
-    step_group = all_days_df.groupby(["day", "step"]).sum(numeric_only=True).reset_index()
-    step_group["global_step"] = (step_group["day"] - 1) * env.num_steps + step_group["step"]
-     
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), sharex=True)
-     
-    # Subplot 1: Grid Import vs P2P Buy
-    ax1.plot(step_group["global_step"], step_group["grid_import_with_p2p"], label="Grid Import (with P2P)", color='r')
-    ax1.plot(step_group["global_step"], step_group["p2p_buy"], label="P2P Buy", color='g')
-    ax1.set_ylabel("Energy (kWh)")
-    ax1.set_title("Community Energy Consumption: Grid Import vs. P2P Buy")
-    ax1.legend()
-    ax1.grid(True, linestyle='--', alpha=0.6)
-
-    # Subplot 2: Grid Export vs P2P Sell
-    ax2.plot(step_group["global_step"], step_group["grid_export"], label="Grid Export", color='orange')
-    ax2.plot(step_group["global_step"], step_group["p2p_sell"], label="P2P Sell", color='b')
-    ax2.set_xlabel("Global Timestep")
-    ax2.set_ylabel("Energy (kWh)")
-    ax2.set_title("Community Energy Generation: Grid Export vs. P2P Sell")
-    ax2.legend()
-    ax2.grid(True, linestyle='--', alpha=0.6)
-     
-    plt.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "combined_energy_flows_timeseries.png"))
-    plt.close()
-
-    # Stacked Bar of Daily Energy Sources
-    daily_agg = all_days_df.groupby("day").sum(numeric_only=True)
-     
-    plt.figure(figsize=(12, 7))
-    plt.bar(daily_agg.index, daily_agg["grid_import_with_p2p"], label="Grid Import (with P2P)", color='crimson')
-    plt.bar(daily_agg.index, daily_agg["p2p_buy"], bottom=daily_agg["grid_import_with_p2p"], label="P2P Buy", color='limegreen')
-    plt.plot(daily_agg.index, daily_agg["grid_import_no_p2p"], label="Baseline Grid Import (No P2P)", color='blue', linestyle='--', marker='o')
-     
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Daily Energy Procurement: Baseline vs. P2P+Grid")
-    plt.xticks(daily_agg.index)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_energy_procurement_stacked.png"))
-    plt.close()
-
-    # Fairness Metrics Over Time
-    plt.figure(figsize=(12, 6))
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_cost_savings"], label="Cost Savings Fairness", marker='o')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_grid_reduction"], label="Grid Reduction Fairness", marker='s')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_reward"], label="Reward Fairness", marker='^')
-    plt.xlabel("Day")
-    plt.ylabel("Jain's Fairness Index")
-    plt.title("Daily Fairness Metrics")
-    plt.xticks(plot_daily_df["day"])
-    plt.ylim(0, 1.05)
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_fairness_metrics.png"))
-    plt.close()
-
-    # Per-House Savings and Reductions
-    fig, ax1 = plt.subplots(figsize=(15, 7))
-     
-    house_ids_str = house_level_df.index.astype(str)
-    bar_width = 0.4
-    index = np.arange(len(house_ids_str))
-
-    # Bar chart for cost savings
-    color1 = 'tab:green'
-    ax1.set_xlabel('House ID')
-    ax1.set_ylabel('Total Cost Savings ($)', color=color1)
-    ax1.bar(index - bar_width/2, house_level_df["cost_savings"], bar_width, label='Cost Savings', color=color1)
-    ax1.tick_params(axis='y', labelcolor=color1)
-    ax1.set_xticks(index)
-    ax1.set_xticklabels(house_ids_str, rotation=45, ha="right")
-     
-    # Second y-axis for grid import reduction
-    ax2 = ax1.twinx()
-    color2 = 'tab:blue'
-    ax2.set_ylabel('Total Grid Import Reduction (kWh)', color=color2)
-    ax2.bar(index + bar_width/2, house_level_df["import_reduction"], bar_width, label='Import Reduction', color=color2)
-    ax2.tick_params(axis='y', labelcolor=color2)
-
-    plt.title(f'Total Cost Savings & Grid Import Reduction Per House (over {days_to_evaluate} days)')
-    fig.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "per_house_summary.png"))
-    plt.close()
-     
-    # Price Dynamics for a Single Day
-    day1_prices = all_days_df[all_days_df['day'] == 1][['step', 'grid_price', 'peer_price']].drop_duplicates()
-    plt.figure(figsize=(12, 6))
-    plt.plot(day1_prices['step'], day1_prices['grid_price'], label='Grid Price', color='darkorange')
-    plt.plot(day1_prices['step'], day1_prices['peer_price'], label='P2P Price', color='teal')
-    plt.xlabel("Timestep of Day")
-    plt.ylabel("Price ($/kWh)")
-    plt.title("Price Dynamics on Day 1")
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.6)
-    plt.savefig(os.path.join(plots_dir, "price_dynamics_day1.png"))
-    plt.close()
-     
-    # Battery State of Charge for Sample Houses
-    day1_df = all_days_df[all_days_df['day'] == 1]
-    battery_houses = day1_df.dropna(subset=['soc'])['house'].unique()
-     
-    if len(battery_houses) > 0:
-        sample_houses = battery_houses[:min(4, len(battery_houses))]
-        plt.figure(figsize=(12, 6))
-        for house in sample_houses:
-            house_df = day1_df[day1_df['house'] == house]
-            plt.plot(house_df['step'], house_df['soc'] * 100, label=f'House {house}')
-         
-        plt.xlabel("Timestep of Day")
-        plt.ylabel("State of Charge (%)")
-        plt.title("Battery SoC on Day 1 for Sample Houses")
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.6)
-        plt.savefig(os.path.join(plots_dir, "soc_dynamics_day1.png"))
-        plt.close()
-
-    print("All plots have been generated and saved. Evaluation complete.")
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/maddpg/maddpg_train.py

@@ -1,382 +0,0 @@
-import os
-import sys
-import re
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-import pandas as pd
-import time
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from maddpg.trainer.maddpg import MADDPG
-
-def main():
-
-    STATE_TO_RUN = "oklahoma"  # "pennsylvania" or "colorado" or "oklahoma"
-
-    # Set the path to your training data
-    DATA_FILE_PATH = "/path/to/project/training/5houses_152days_TRAIN.csv"
-    num_episodes = 10000
-    batch_size = 256
-    checkpoint_interval = 100000
-    window_size = 32
-
-    env = SolarSys(
-        data_path=DATA_FILE_PATH,
-        state=STATE_TO_RUN,
-        time_freq="15T"
-    )
-
-    # Sanity check: env I/O shapes
-    print("Observation space:", env.observation_space)
-    print("Action space     :", env.action_space)
-
-    # Reset and inspect obs
-    obs = env.reset()
-    print(f"Reset returned {len(obs)} agent observations; each obs shape: {np.array(obs).shape}")
-
-    # Sample random actions and do one step
-    dummy_actions = np.random.rand(env.num_agents, env.action_space.shape[1]).astype(np.float32)
-    next_obs, rewards, done, info = env.step(dummy_actions)
-    print(f"Step outputs → next_obs: {len(next_obs)}×{np.array(next_obs).shape[1]}, "
-          f"rewards: {len(rewards)}, done: {done}")
-    print("Info keys:", list(info.keys()))
-
-    # Count the number of houses in each group
-    env.group_counts = {
-        0: env.agent_groups.count(0),
-        1: env.agent_groups.count(1)
-    }
-    print(f"Number of houses in each group: {env.group_counts}")
-
-    max_steps = env.num_steps
-
-    # Dims from the env
-    num_agents = env.num_agents
-    local_state_dim = env.observation_space.shape[1]
-    action_dim = env.action_space.shape[1]
-
-    # Build a unique run directory
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"maddpg_para_sharing_{STATE_TO_RUN}_{num_agents}agents_{num_episodes}eps_{timestamp}"
-    root_dir = os.path.join("FINALE_FINALE_FINALE", run_name)
-    os.makedirs(root_dir, exist_ok=True)
-    print(f"Saving training outputs to: {root_dir}")
-
-    logs_dir = os.path.join(root_dir, "logs")
-    plots_dir = os.path.join(root_dir, "plots")
-    os.makedirs(logs_dir, exist_ok=True)
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Create the MADDPG agent
-    maddpg = MADDPG(
-        num_agents=num_agents,
-        state_dim=local_state_dim,
-        action_dim=action_dim,
-        gamma=0.95,
-        tau=0.01,
-        lr_actor=1e-4,
-        lr_critic=1e-3,
-        buffer_size=1000000,
-        noise_episodes=5000,
-        init_sigma=0.3,
-        final_sigma=0.01,
-        batch_size=batch_size
-    )
-
-    # Tracking / Logging Variables
-    episode_rewards = []
-    episode_total_rewards = []
-    block_mean_rewards = []
-    block_total_rewards = []
-
-    agent_rewards_log = [[] for _ in range(num_agents)]
-    best_mean_reward = -1e9
-    best_model_path = os.path.join(logs_dir, "best_model.pth")
-
-    daily_rewards = []
-    monthly_rewards = []
-
-    training_start_time = time.time()
-    episode_durations = []
-    total_steps_global = 0
-    episode_log_data = []
-    performance_metrics_log = []
-
-    agent_charge_log = [[] for _ in range(num_agents)]
-    agent_discharge_log = [[] for _ in range(num_agents)]
-
-    # Training Loop
-    for episode in range(1, num_episodes + 1):
-        episode_start_time = time.time()
-
-        obs = np.array(env.reset(), dtype=np.float32)
-
-        # Collect metrics from the previous episode
-        if episode > 1:
-            last_episode_metrics = env.get_episode_metrics()
-            last_episode_metrics['Episode'] = episode - 1
-            performance_metrics_log.append(last_episode_metrics)
-
-        total_reward = np.zeros(num_agents, dtype=np.float32)
-        done = False
-        step_count = 0
-        day_logs = []
-        episode_charges = [[] for _ in range(num_agents)]
-        episode_discharges = [[] for _ in range(num_agents)]
-
-        while not done:
-            # Select actions using the MADDPG agent
-            actions = maddpg.select_actions(obs)
-
-            # Step environment
-            next_obs_list, rewards, done, info = env.step(actions)
-            next_obs = np.array(next_obs_list, dtype=np.float32)
-
-            # Store the transition in the replay buffer
-            maddpg.store_transition(obs, actions, rewards, next_obs, done)
-
-            # Train the agent at every step
-            maddpg.train()
-
-            total_reward += rewards
-            obs = next_obs
-            step_count += 1
-            total_steps_global += 1
-            
-            for i in range(num_agents):
-                episode_charges[i].append(info["charge_amount"][i])
-                episode_discharges[i].append(info["discharge_amount"][i])
-
-            day_logs.append({
-                "step": step_count - 1,
-                "grid_import_no_p2p": info["grid_import_no_p2p"],
-                "grid_import_with_p2p": info["grid_import_with_p2p"],
-                "p2p_buy": info["p2p_buy"],
-                "p2p_sell": info["p2p_sell"],
-                "costs": info["costs"],
-                "charge_amount": info.get("charge_amount", np.zeros(num_agents)),
-                "discharge_amount": info.get("discharge_amount", np.zeros(num_agents))
-            })
-
-            if step_count >= max_steps:
-                break
-
-        # After each episode
-        # Compute per-episode metrics
-        sum_ep_reward = float(np.sum(total_reward))
-        mean_ep_reward = float(np.mean(total_reward))
-
-        episode_total_rewards.append(sum_ep_reward)
-        episode_rewards.append(mean_ep_reward)
-        daily_rewards.append(mean_ep_reward)
-
-        # If we just finished a block of window_size episodes, aggregate
-        if len(daily_rewards) % window_size == 0:
-            last_totals = episode_total_rewards[-window_size:]
-            block_sum = sum(last_totals)
-            block_total_rewards.append(block_sum)
-
-            last_means = daily_rewards[-window_size:]
-            block_mean = sum(last_means) / window_size
-            block_mean_rewards.append(block_mean)
-
-            block_idx = len(block_mean_rewards)
-            print(
-                f"→ Completed Block {block_idx} "
-                f"| Episodes {(block_idx-1)*window_size + 1}–{block_idx*window_size} "
-                f"| Block Total Reward: {block_sum:.3f} "
-                f"| Block Mean Reward: {block_mean:.3f}"
-            )
-
-        # Log agent-level rewards
-        for i in range(num_agents):
-            agent_rewards_log[i].append(total_reward[i])
-            agent_charge_log[i].append(np.mean(episode_charges[i]))
-            agent_discharge_log[i].append(np.mean(episode_discharges[i]))
-            
-        # Summarize P2P steps
-        steps_data = []
-        for entry in day_logs:
-            step_idx = entry["step"]
-            p2p_buy_array = entry["p2p_buy"]
-            p2p_sell_array = entry["p2p_sell"]
-            grid_no_p2p_array = entry["grid_import_no_p2p"]
-            grid_with_p2p_array = entry["grid_import_with_p2p"]
-
-            steps_data.append({
-                "step": step_idx,
-                "p2p_buy_sum": float(np.sum(p2p_buy_array)),
-                "p2p_sell_sum": float(np.sum(p2p_sell_array)),
-                "grid_import_no_p2p_sum": float(np.sum(grid_no_p2p_array)),
-                "grid_import_with_p2p_sum": float(np.sum(grid_with_p2p_array))
-            })
-
-        baseline_cost = np.sum([np.sum(entry["grid_import_no_p2p"]) * env.get_grid_price(entry["step"])
-                                for entry in day_logs])
-        actual_cost = np.sum([np.sum(entry["costs"]) for entry in day_logs])
-        cost_reduction = (baseline_cost - actual_cost) / baseline_cost
-
-        # Call on_episode_end() for noise decay schedule
-        maddpg.on_episode_end()
-        
-        # Save if best
-        if mean_ep_reward > best_mean_reward:
-            best_mean_reward = mean_ep_reward
-            maddpg.save(best_model_path)
-
-        if episode % checkpoint_interval == 0:
-            ckpt_path = os.path.join(logs_dir, f"checkpoint_{episode}.pth")
-            maddpg.save(ckpt_path)
-
-        episode_end_time = time.time()
-        episode_duration = episode_end_time - episode_start_time
-
-        print(
-            f"Episode {episode}/{num_episodes} "
-            f"| Time per Episode: {episode_duration:.2f}s "
-            f"| Steps: {step_count} "
-            f"| Mean Reward: {mean_ep_reward:.3f} "
-            f"| Cost Reduction: {cost_reduction:.2%}"
-        )
-
-        # Record data in per-episode log
-        episode_log_data.append({
-            "Episode": episode,
-            "Steps": step_count,
-            "Mean_Reward": mean_ep_reward,
-            "Total_Reward": sum_ep_reward,
-            "Cost_Reduction_Pct": cost_reduction * 100,
-            "Baseline_Cost": baseline_cost,
-            "Actual_Cost": actual_cost,
-            "Episode_Duration": episode_duration,
-            "Total_Charge": np.sum([np.sum(entry["charge_amount"]) for entry in day_logs]),
-            "Total_Discharge": np.sum([np.sum(entry["discharge_amount"]) for entry in day_logs])
-        })
-
-        for i in range(num_agents):
-            agent_charge_log[i].append(np.mean(episode_charges[i]))
-            agent_discharge_log[i].append(np.mean(episode_discharges[i]))
-
-    # Capture the final episode's metrics
-    final_episode_metrics = env.get_episode_metrics()
-    final_episode_metrics['Episode'] = num_episodes
-    performance_metrics_log.append(final_episode_metrics)
-
-    # End of all training
-    training_end_time = time.time()
-    total_training_time = training_end_time - training_start_time
-
-    # Save out per-episode agent rewards + mean rewards
-    np.save(os.path.join(logs_dir, "agent_rewards.npy"), np.array(agent_rewards_log))
-    np.save(os.path.join(logs_dir, "mean_rewards.npy"), np.array(episode_rewards))
-    np.save(os.path.join(logs_dir, "total_rewards.npy"), np.array(episode_total_rewards))
-
-    # Create Final DataFrame for Logging and Plotting
-    df_rewards_log = pd.DataFrame(episode_log_data)
-    df_perf_log = pd.DataFrame(performance_metrics_log)
-
-    # Merge the two DataFrames on the 'Episode' column
-    df_final_log = pd.merge(df_rewards_log, df_perf_log.drop(columns=[
-        'degradation_cost_over_time',
-        'cost_savings_over_time',
-        'grid_reduction_over_time'
-    ]), on="Episode")
-
-    # PLOTTING
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Helper: centered moving average
-    def moving_avg(series, window):
-        return pd.Series(series).rolling(window=window, center=True, min_periods=1).mean().to_numpy()
-
-    # Smoothing window (in episodes)
-    ma_window = 300
-    episodes = np.arange(1, num_episodes + 1)
-
-    # Mean Reward moving average
-    reward_ma = moving_avg(df_final_log["Mean_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, reward_ma, linewidth=2, label=f"Mean Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Mean Reward")
-    plt.title("MADDPG: Mean Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "mean_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Total Reward moving average
-    total_ma = moving_avg(df_final_log["Total_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, total_ma, linewidth=2, label=f"Total Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Total Reward")
-    plt.title("MADDPG: Total Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "total_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Cost Reduction (%) moving average
-    cost_ma = moving_avg(df_final_log["Cost_Reduction_Pct"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, cost_ma, linewidth=2, label="Cost Reduction MA (%)")
-    plt.xlabel("Episode")
-    plt.ylabel("Cost Reduction (%)")
-    plt.title("MADDPG: Cost Reduction Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "cost_reduction_ma.png"), dpi=200)
-    plt.close()
-
-    # Battery Degradation Cost moving average
-    degradation_ma = moving_avg(df_final_log["battery_degradation_cost_total"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, degradation_ma, linewidth=2, label=f"Degradation Cost MA (win={ma_window})", color='purple')
-    plt.xlabel("Episode")
-    plt.ylabel("Total Degradation Cost ($)")
-    plt.title("MADDPG: Battery Degradation Cost Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "degradation_cost_ma.png"), dpi=200)
-    plt.close()
-
-    print(f"\nAll moving-average plots saved to: {plots_dir}")
-
-    # Save Final Logs to CSV
-    total_time_row = pd.DataFrame([{
-        "Episode": "Total_Training_Time",
-        "Episode_Duration": total_training_time
-    }])
-    df_to_save = pd.concat([df_final_log, total_time_row], ignore_index=True)
-
-    log_csv_path = os.path.join(logs_dir, "training_performance_log.csv")
-
-    # Select and reorder columns for the final CSV
-    columns_to_save = [
-        "Episode",
-        "Mean_Reward",
-        "Total_Reward",
-        "Cost_Reduction_Pct",
-        "Episode_Duration",
-        "battery_degradation_cost_total",
-    ]
-    df_to_save = df_to_save[columns_to_save]
-
-    df_to_save.to_csv(log_csv_path, index=False)
-
-    print(f"Saved comprehensive training performance log to: {log_csv_path}")
-
-    # Final Timings Printout
-    print("\n" + "="*50)
-    print("TRAINING COMPLETE".center(50))
-    print(f"Total training time: {total_training_time:.2f} seconds")
-    print("="*50)
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/maddpg/trainer/maddpg.py

@@ -1,196 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.optim as optim
-import numpy as np
-import random
-from collections import deque
-from torch.utils.data import Dataset, DataLoader
-
-class ReplayBufferDataset(Dataset):
-    def __init__(self, max_size=100000):
-        self.buffer = deque(maxlen=max_size)
-
-    def add(self, states, actions, rewards, next_states, done):
-        data = (
-            states,
-            actions,
-            np.array(rewards, dtype=np.float32),
-            next_states,
-            np.float32(done)
-        )
-        self.buffer.append(data)
-
-    def __len__(self):
-        return len(self.buffer)
-
-    def __getitem__(self, idx):
-        states, actions, rewards, next_states, done = self.buffer[idx]
-        return (
-            torch.from_numpy(states),
-            torch.from_numpy(actions),
-            torch.from_numpy(rewards),
-            torch.from_numpy(next_states),
-            torch.tensor(done, dtype=torch.float32)
-        )
-
-class Actor(nn.Module):
-    def __init__(self, state_dim, action_dim, hidden_dim=64):
-        super(Actor, self).__init__()
-        self.net = nn.Sequential(
-            nn.Linear(state_dim, hidden_dim),
-            nn.ReLU(),
-            nn.Linear(hidden_dim, hidden_dim),
-            nn.ReLU(),
-            nn.Linear(hidden_dim, action_dim),
-            nn.Sigmoid()
-        )
-
-    def forward(self, state):
-        return self.net(state)
-
-class SharedCritic(nn.Module):
-    def __init__(self, global_state_dim, global_action_dim, hidden_dim=128, num_agents=1):
-        super().__init__()
-        self.net = nn.Sequential(
-            nn.Linear(global_state_dim + global_action_dim, hidden_dim),
-            nn.ReLU(),
-            nn.Linear(hidden_dim, hidden_dim),
-            nn.ReLU(),
-            nn.Linear(hidden_dim, num_agents)
-        )
-
-    def forward(self, global_state, global_action):
-        x = torch.cat([global_state, global_action], dim=1)
-        return self.net(x)
-
-class Agent:
-    def __init__(self, local_state_dim, action_dim, lr_actor=1e-3, device=torch.device('cpu')):
-        self.device = device
-        self.actor = Actor(local_state_dim, action_dim).to(device)
-        self.target_actor = Actor(local_state_dim, action_dim).to(device)
-        self.actor_optim = optim.Adam(self.actor.parameters(), lr=lr_actor)
-        self.target_actor.load_state_dict(self.actor.state_dict())
-
-    def sync_target(self, tau):
-        for tp, p in zip(self.target_actor.parameters(), self.actor.parameters()):
-            tp.data.copy_(tau * p.data + (1.0 - tau) * tp.data)
-
-class MADDPG:
-    def __init__(self, num_agents, local_state_dim, action_dim,
-                 gamma=0.95, tau=0.01, lr_actor=1e-4, lr_critic=1e-3,
-                 buffer_size=100000, noise_episodes=100, init_sigma=0.2, final_sigma=0.01,
-                 batch_size=128, num_workers=0):
-
-        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        self.num_agents = num_agents
-        self.gamma = gamma
-        self.tau = tau
-        self.init_sigma = init_sigma
-        self.final_sigma = final_sigma
-        self.noise_episodes = noise_episodes
-        self.current_episode = 0
-
-        self.actor = Actor(local_state_dim, action_dim).to(self.device)
-        self.target_actor = Actor(local_state_dim, action_dim).to(self.device)
-        self.target_actor.load_state_dict(self.actor.state_dict())
-        self.actor_optim = optim.Adam(self.actor.parameters(), lr=lr_actor)
-
-        global_state_dim = num_agents * local_state_dim
-        global_action_dim = num_agents * action_dim
-        self.critic = SharedCritic(global_state_dim, global_action_dim, num_agents=num_agents).to(self.device)
-        self.target_critic = SharedCritic(global_state_dim, global_action_dim, num_agents=num_agents).to(self.device)
-        self.target_critic.load_state_dict(self.critic.state_dict())
-        self.critic_optim = optim.Adam(self.critic.parameters(), lr=lr_critic)
-
-        self.batch_size = batch_size
-        self.num_workers = num_workers
-        self.memory = ReplayBufferDataset(max_size=buffer_size)
-        self.dataloader = None
-        self.loader_iter = None
-
-    def select_actions(self, states, evaluate=False):
-        states_t = torch.as_tensor(states, dtype=torch.float32, device=self.device)
-        with torch.no_grad():
-            actions_t = torch.stack([
-                self.actor(states_t[i]) for i in range(self.num_agents)
-            ], dim=0)
-        actions = actions_t.cpu().numpy()
-
-        if not evaluate:
-            frac = min(float(self.current_episode) / self.noise_episodes, 1.0)
-            current_sigma = self.init_sigma - frac * (self.init_sigma - self.final_sigma)
-            noise = np.random.normal(0, current_sigma, size=actions.shape)
-            actions += noise
-        return np.clip(actions, 0.0, 1.0)
-
-    def store_transition(self, states, actions, rewards, next_states, done):
-        self.memory.add(states, actions, rewards, next_states, done)
-
-    def train(self):
-        if len(self.memory) < self.batch_size:
-            return
-        
-        should_pin_memory = self.device.type == 'cuda'
-        if self.dataloader is None:
-            self.dataloader = DataLoader(self.memory, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers, pin_memory=should_pin_memory, drop_last=True)
-            self.loader_iter = iter(self.dataloader)
-        try:
-            s, a, r, s2, d = next(self.loader_iter)
-        except StopIteration:
-            self.dataloader = DataLoader(self.memory, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers, pin_memory=should_pin_memory, drop_last=True)
-            self.loader_iter = iter(self.dataloader)
-            s, a, r, s2, d = next(self.loader_iter)
-        
-        s_t, a_t, r_t, s2_t, d_t = s.to(self.device), a.to(self.device), r.to(self.device), s2.to(self.device), d.to(self.device).unsqueeze(-1)
-        r_t = (r_t - r_t.mean()) / (r_t.std() + 1e-7)
-        batch_len = s_t.shape[0]
-        gs, ga, ns = s_t.reshape(batch_len, -1), a_t.reshape(batch_len, -1), s2_t.reshape(batch_len, -1)
-
-        with torch.no_grad():
-            targ_actions = torch.cat([self.target_actor(s2_t[:, i, :]) for i in range(self.num_agents)], dim=1)
-            Q_prime = self.target_critic(ns, targ_actions)
-            targets = r_t + self.gamma * (1 - d_t) * Q_prime
-        Q = self.critic(gs, ga)
-        critic_loss = nn.MSELoss()(Q, targets)
-        self.critic_optim.zero_grad()
-        critic_loss.backward()
-        torch.nn.utils.clip_grad_norm_(self.critic.parameters(), 1.0)
-        self.critic_optim.step()
-
-        all_actions = torch.cat([self.actor(s_t[:, i, :]) for i in range(self.num_agents)], dim=1)
-        actor_loss = -self.critic(gs, all_actions).mean()
-        
-        self.actor_optim.zero_grad()
-        actor_loss.backward()
-        torch.nn.utils.clip_grad_norm_(self.actor.parameters(), 1.0)
-        self.actor_optim.step()
-
-        for tp, p in zip(self.target_actor.parameters(), self.actor.parameters()):
-            tp.data.copy_(self.tau * p.data + (1.0 - self.tau) * tp.data)
-        for tp, p in zip(self.target_critic.parameters(), self.critic.parameters()):
-            tp.data.copy_(self.tau * p.data + (1.0 - self.tau) * tp.data)
-
-    def on_episode_end(self):
-        self.current_episode += 1
-
-    def save(self, path: str):
-        payload = {
-            "critic": self.critic.state_dict(),
-            "target_critic": self.target_critic.state_dict(),
-            "critic_optim": self.critic_optim.state_dict(),
-            "actor": self.actor.state_dict(),
-            "target_actor": self.target_actor.state_dict(),
-            "actor_optim": self.actor_optim.state_dict(),
-            "current_episode": self.current_episode,
-        }
-        torch.save(payload, path)
-
-    def load(self, path: str):
-        checkpoint = torch.load(path, map_location=self.device)
-        self.critic.load_state_dict(checkpoint["critic"])
-        self.target_critic.load_state_dict(checkpoint["target_critic"])
-        self.critic_optim.load_state_dict(checkpoint["critic_optim"])
-        self.actor.load_state_dict(checkpoint["actor"])
-        self.target_actor.load_state_dict(checkpoint["target_actor"])
-        self.actor_optim.load_state_dict(checkpoint["actor_optim"])
-        self.current_episode = checkpoint.get("current_episode", 0)

Other_algorithms/Flat_System/mappo/mappo_evaluation.py

@@ -1,500 +0,0 @@
-# mappo_evaluate.py
-import os
-import sys
-import time
-import re
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import torch
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from mappo.trainer.mappo import MAPPO
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-def compute_jains_fairness(values: np.ndarray) -> float:
-    if len(values) == 0:
-        return 0.0
-    if np.all(values == 0):
-        return 1.0
-    num = (values.sum())**2
-    den = len(values) * (values**2).sum()
-    return num / den
-
-def main():
-    # User parameters
-    MODEL_PATH = "/path/to/project/mappo_pennsylvania_100agents_10000eps/logs/best_model.pth"
-    DATA_PATH = "/path/to/project/testing/100houses_30days_TEST.csv"
-    DAYS_TO_EVALUATE = 30
-
-    model_path = MODEL_PATH
-    data_path = DATA_PATH
-    days_to_evaluate = DAYS_TO_EVALUATE
-    SOLAR_THRESHOLD = 0.1
-
-    state_match = re.search(r"mappo_(oklahoma|colorado|pennsylvania)_", model_path)
-    if not state_match:
-        raise ValueError(
-            "Could not automatically detect the state (oklahoma, colorado, or pennsylvania) "
-            "from the model path. Please ensure your model's parent folder is named correctly, "
-            "e.g., 'mappo_oklahoma_...'"
-        )
-    detected_state = state_match.group(1)
-    print(f"--- Detected state: {detected_state.upper()} ---")
-
-    # Env setup
-    env = SolarSys(
-        data_path=data_path,
-        state=detected_state,
-        time_freq="3H"
-    )
-    eval_steps = env.num_steps
-    house_ids = env.house_ids
-    num_agents = env.num_agents
-
-    # Generate a unique eval run folder
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"eval_mappo_{num_agents}agents_{days_to_evaluate}days_{timestamp}"
-    output_folder = os.path.join("runs_with_battery", run_name)
-    logs_dir = os.path.join(output_folder, "logs")
-    plots_dir = os.path.join(output_folder, "plots")
-    for d in (logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Saving evaluation outputs to: {output_folder}")
-        
-    local_dim = env.observation_space.shape[1]
-    global_dim = num_agents * local_dim
-    act_dim = env.action_space.shape[1]
-    
-    mappo = MAPPO(
-        n_agents=num_agents,
-        local_dim=local_dim,
-        global_dim=global_dim,
-        act_dim=act_dim,
-        lr=2e-4,
-        gamma=0.95,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=10,
-        batch_size=1024
-    )
-
-    # Load MAPPO checkpoint
-    mappo.load(model_path)
-    mappo.actor.to(device).eval()
-    mappo.critic.to(device).eval()
-
-    # Prepare logs
-    all_logs = []
-    daily_summaries = []
-    step_timing_list = []
-     
-    evaluation_start = time.time()
-
-    for day_idx in range(days_to_evaluate):
-        obs = env.reset()
-        done = False
-        step_count = 0
-        day_logs = []
-
-        while not done:
-            step_start_time = time.time()
-            global_obs = np.array(obs).flatten()
-
-            # Select actions with MAPPO
-            actions, _ = mappo.select_action(obs, global_obs)
-
-            next_obs, rewards, done, info = env.step(actions)
-             
-            # Consolidated Logging
-            step_end_time = time.time()
-            step_duration = step_end_time - step_start_time
-             
-            print(f"[Day {day_idx+1}, Step {step_count}] Step time: {step_duration:.6f} seconds")
-
-            step_timing_list.append({
-                "day": day_idx + 1,
-                "step": step_count,
-                "step_time_s": step_duration
-            })
-
-            grid_price_now = env.get_grid_price(step_count)
-            peer_price_now = info.get("peer_price", env.get_peer_price(step_count,
-                                float(info["p2p_sell"].sum()),
-                                float(info["p2p_buy"].sum())))
-
-            for i, hid in enumerate(house_ids):
-                is_battery_house = hid in env.batteries
-                p2p_buy = float(info["p2p_buy"][i])
-                p2p_sell = float(info["p2p_sell"][i])
-                charge_amount = float(info.get("charge_amount", [0]*num_agents)[i])
-                discharge_amount = float(info.get("discharge_amount", [0]*num_agents)[i])
-
-                day_logs.append({
-                    "day": day_idx + 1,
-                    "step": step_count,
-                    "house": hid,
-                    "grid_import_no_p2p": float(info["grid_import_no_p2p"][i]),
-                    "grid_import_with_p2p": float(info["grid_import_with_p2p"][i]),
-                    "grid_export": float(info.get("grid_export", [0]*num_agents)[i]),
-                    "p2p_buy": p2p_buy,
-                    "p2p_sell": p2p_sell,
-                    "actual_cost": float(info["costs"][i]),
-                    "baseline_cost": float(info["grid_import_no_p2p"][i]) * grid_price_now,
-                    "total_demand": float(env.demands[hid][step_count]),
-                    "total_solar": float(env.solars[hid][step_count]),
-                    "grid_price": grid_price_now,
-                    "peer_price": peer_price_now,
-                    "soc": (env.batteries[hid]["soc"] / env.batteries[hid]["max_capacity"]) if is_battery_house else np.nan,
-                    "degradation_cost": ((charge_amount + discharge_amount) * env.batteries[hid]["degradation_cost_per_kwh"]) if is_battery_house else 0.0,
-                    "reward": float(rewards[i]),
-                })
-
-            obs = next_obs
-            step_count += 1
-            if step_count >= eval_steps:
-                break
-         
-        day_df = pd.DataFrame(day_logs)
-        all_logs.extend(day_logs)
-
-        # Consolidated daily summary calculation
-        grouped_house = day_df.groupby("house").sum(numeric_only=True)
-        grouped_step = day_df.groupby("step").sum(numeric_only=True)
-
-        total_demand = grouped_step["total_demand"].sum()
-        total_solar = grouped_step["total_solar"].sum()
-        total_p2p_buy = grouped_house["p2p_buy"].sum()
-        total_p2p_sell = grouped_house["p2p_sell"].sum()
-
-        baseline_cost_per_house = grouped_house["baseline_cost"]
-        actual_cost_per_house = grouped_house["actual_cost"]
-        cost_savings_per_house = baseline_cost_per_house - actual_cost_per_house
-        day_total_cost_savings = cost_savings_per_house.sum()
-         
-        if baseline_cost_per_house.sum() > 0:
-            overall_cost_savings_pct = day_total_cost_savings / baseline_cost_per_house.sum()
-        else:
-            overall_cost_savings_pct = 0.0
-
-        baseline_import_per_house = grouped_house["grid_import_no_p2p"]
-        actual_import_per_house = grouped_house["grid_import_with_p2p"]
-        import_reduction_per_house = baseline_import_per_house - actual_import_per_house
-        day_total_import_reduction = import_reduction_per_house.sum()
-         
-        if baseline_import_per_house.sum() > 0:
-            overall_import_reduction_pct = day_total_import_reduction / baseline_import_per_house.sum()
-        else:
-            overall_import_reduction_pct = 0.0
-
-        fairness_cost_savings = compute_jains_fairness(cost_savings_per_house.values)
-        fairness_import_reduction = compute_jains_fairness(import_reduction_per_house.values)
-        fairness_rewards = compute_jains_fairness(grouped_house["reward"].values)
-        fairness_p2p_buy = compute_jains_fairness(grouped_house["p2p_buy"].values)
-        fairness_p2p_sell = compute_jains_fairness(grouped_house["p2p_sell"].values)
-        fairness_p2p_total = compute_jains_fairness((grouped_house["p2p_buy"] + grouped_house["p2p_sell"]).values)
-        day_total_degradation_cost = grouped_house["degradation_cost"].sum()
-
-        daily_summaries.append({
-            "day": day_idx + 1,
-            "day_total_demand": total_demand,
-            "day_total_solar": total_solar,
-            "day_p2p_buy": total_p2p_buy,
-            "day_p2p_sell": total_p2p_sell,
-            "cost_savings_abs": day_total_cost_savings,
-            "cost_savings_pct": overall_cost_savings_pct,
-            "fairness_cost_savings": fairness_cost_savings,
-            "grid_reduction_abs": day_total_import_reduction,
-            "grid_reduction_pct": overall_import_reduction_pct,
-            "fairness_grid_reduction": fairness_import_reduction,
-            "fairness_reward": fairness_rewards,
-            "fairness_p2p_buy": fairness_p2p_buy,
-            "fairness_p2p_sell": fairness_p2p_sell,
-            "fairness_p2p_total": fairness_p2p_total,
-            "total_degradation_cost": day_total_degradation_cost
-        })
-
-    # Final processing and saving
-    evaluation_end = time.time()
-    total_eval_time = evaluation_end - evaluation_start
-    print(f"\nEvaluation loop finished. Total time: {total_eval_time:.2f} seconds.")
-
-    all_days_df = pd.DataFrame(all_logs)
-    combined_csv_path = os.path.join(logs_dir, "step_logs_all_days.csv")
-    all_days_df.to_csv(combined_csv_path, index=False)
-    print(f"Saved combined step-level logs to: {combined_csv_path}")
-
-    step_timing_df = pd.DataFrame(step_timing_list)
-    timing_csv_path = os.path.join(logs_dir, "step_timing_log.csv")
-    step_timing_df.to_csv(timing_csv_path, index=False)
-    print(f"Saved step timing logs to: {timing_csv_path}")
-
-    house_level_df = all_days_df.groupby("house").agg({
-        "baseline_cost": "sum",
-        "actual_cost": "sum",
-        "grid_import_no_p2p": "sum",
-        "grid_import_with_p2p": "sum",
-        "degradation_cost": "sum"
-    })
-    house_level_df["cost_savings"] = house_level_df["baseline_cost"] - house_level_df["actual_cost"]
-    house_level_df["import_reduction"] = house_level_df["grid_import_no_p2p"] - house_level_df["grid_import_with_p2p"]
-     
-    house_summary_csv = os.path.join(logs_dir, "summary_per_house.csv")
-    house_level_df.to_csv(house_summary_csv)
-    print(f"Saved final summary per house to: {house_summary_csv}")
-
-    fairness_grid_all = compute_jains_fairness(house_level_df["import_reduction"].values)
-    fairness_cost_all = compute_jains_fairness(house_level_df["cost_savings"].values)
-     
-    daily_summary_df = pd.DataFrame(daily_summaries)
-
-    total_cost_savings_all = daily_summary_df["cost_savings_abs"].sum()
-    total_baseline_cost_all = all_days_df.groupby('day')['baseline_cost'].sum().sum()
-    pct_cost_savings_all = total_cost_savings_all / total_baseline_cost_all if total_baseline_cost_all > 0 else 0.0
-
-    total_grid_reduction_all = daily_summary_df["grid_reduction_abs"].sum()
-    total_baseline_import_all = all_days_df.groupby('day')['grid_import_no_p2p'].sum().sum()
-    pct_grid_reduction_all = total_grid_reduction_all / total_baseline_import_all if total_baseline_import_all > 0 else 0.0
-
-    total_degradation_cost_all = daily_summary_df["total_degradation_cost"].sum()
-
-    # Calculate alternative performance metrics
-
-    # Grid Reduction During Solar Hours
-    agg_solar_per_step = all_days_df.groupby(['day', 'step'])['total_solar'].sum()
-    sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents)
-    sunny_df = all_days_df.set_index(['day', 'step'])[sunny_steps_mask].reset_index()
-    baseline_import_sunny = sunny_df['grid_import_no_p2p'].sum()
-    actual_import_sunny = sunny_df['grid_import_with_p2p'].sum()
-    grid_reduction_sunny_pct = 0.0
-    if baseline_import_sunny > 0:
-        grid_reduction_sunny_pct = (baseline_import_sunny - actual_import_sunny) / baseline_import_sunny
-
-    # Community Sourcing Rate
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    total_procured_energy = total_p2p_buy + total_actual_grid_import
-    community_sourcing_rate_pct = 0.0
-    if total_procured_energy > 0:
-        community_sourcing_rate_pct = total_p2p_buy / total_procured_energy
-
-    # Solar Sharing Efficiency
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    total_excess_solar = total_p2p_sell + total_grid_export
-    solar_sharing_efficiency_pct = 0.0
-    if total_excess_solar > 0:
-        solar_sharing_efficiency_pct = total_p2p_sell / total_excess_solar
-
-    # Cost savings in sunny hours
-    baseline_cost_sunny = sunny_df['baseline_cost'].sum()
-    actual_cost_sunny = sunny_df['actual_cost'].sum()
-    cost_savings_sunny_pct = (baseline_cost_sunny - actual_cost_sunny) / baseline_cost_sunny if baseline_cost_sunny > 0 else 0.0
-
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    community_sourcing_rate_pct = total_p2p_buy / (total_p2p_buy + total_actual_grid_import) if (total_p2p_buy + total_actual_grid_import) > 0 else 0.0
-
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    solar_sharing_efficiency_pct = total_p2p_sell / (total_p2p_sell + total_grid_export) if (total_p2p_sell + total_grid_export) > 0 else 0.0
-
-    final_row = {
-        "day": "ALL_DAYS_SUMMARY",
-        "cost_savings_abs": total_cost_savings_all,
-        "cost_savings_pct": pct_cost_savings_all,
-        "grid_reduction_abs": total_grid_reduction_all,
-        "grid_reduction_pct": pct_grid_reduction_all,
-        "fairness_cost_savings": fairness_cost_all,
-        "fairness_grid_reduction": fairness_grid_all,
-        "total_degradation_cost": total_degradation_cost_all,
-        "grid_reduction_sunny_hours_pct": grid_reduction_sunny_pct,
-        "community_sourcing_rate_pct": community_sourcing_rate_pct,
-        "solar_sharing_efficiency_pct": solar_sharing_efficiency_pct,
-    }
-     
-    for col in daily_summary_df.columns:
-        if col not in final_row:
-            final_row[col] = np.nan
-    final_row_df = pd.DataFrame([final_row])
-
-    daily_summary_df = pd.concat([daily_summary_df, final_row_df], ignore_index=True)
-    summary_csv = os.path.join(logs_dir, "summary_per_day.csv")
-    daily_summary_df.to_csv(summary_csv, index=False)
-    print(f"Saved day-level summary with final multi-day row to: {summary_csv}")
-
-    # Final summary printout
-    print("\n================== EVALUATION SUMMARY ==================")
-    print(f"Evaluation finished for {days_to_evaluate} days.\n")
-     
-    print("--- Standard Metrics (24-Hour Average) ---")
-    print(f"Total grid reduction: {total_grid_reduction_all:.2f} kWh ({pct_grid_reduction_all:.2%})")
-    print(f"Total cost savings: ${total_cost_savings_all:.2f} ({pct_cost_savings_all:.2%})")
-    print(f"Jain's fairness on grid reduction: {fairness_grid_all:.3f}")
-    print(f"Jain's fairness on cost savings:   {fairness_cost_all:.3f}\n")
-
-    print("--- Alternative Metrics (Highlighting Peak Performance) ---")
-    print(f"Grid reduction during solar hours: {grid_reduction_sunny_pct:.2%}")
-    print(f"Cost savings during solar hours:   {cost_savings_sunny_pct:.2%}") 
-    print(f"Community sourcing rate:           {community_sourcing_rate_pct:.2%}")
-    print(f"Solar sharing efficiency:          {solar_sharing_efficiency_pct:.2%}")
-     
-    print("=========================================================")
-
-    # Plots
-    plot_daily_df = daily_summary_df[daily_summary_df["day"] != "ALL_DAYS_SUMMARY"].copy()
-    plot_daily_df["day"] = plot_daily_df["day"].astype(int)
-
-    # Daily Cost Savings Percentage
-    plt.figure(figsize=(12, 6))
-    plt.bar(plot_daily_df["day"], plot_daily_df["cost_savings_pct"] * 100, color='skyblue')
-    plt.xlabel("Day")
-    plt.ylabel("Cost Savings (%)")
-    plt.title("Daily Community Cost Savings Percentage")
-    plt.xticks(plot_daily_df["day"])
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_cost_savings_percentage.png"))
-    plt.close()
-
-    # Daily Total Demand vs. Solar
-    plt.figure(figsize=(12, 6))
-    bar_width = 0.4
-    days = plot_daily_df["day"]
-    plt.bar(days - bar_width/2, plot_daily_df["day_total_demand"], width=bar_width, label="Total Demand", color='coral')
-    plt.bar(days + bar_width/2, plot_daily_df["day_total_solar"], width=bar_width, label="Total Solar Generation", color='gold')
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Total Community Demand vs. Solar Generation Per Day")
-    plt.xticks(days)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_demand_vs_solar.png"))
-    plt.close()
-
-    # Combined Time Series of Energy Flows
-    step_group = all_days_df.groupby(["day", "step"]).sum(numeric_only=True).reset_index()
-    step_group["global_step"] = (step_group["day"] - 1) * env.num_steps + step_group["step"]
-     
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), sharex=True)
-     
-    # Subplot 1: Grid Import vs P2P Buy
-    ax1.plot(step_group["global_step"], step_group["grid_import_with_p2p"], label="Grid Import (with P2P)", color='r')
-    ax1.plot(step_group["global_step"], step_group["p2p_buy"], label="P2P Buy", color='g')
-    ax1.set_ylabel("Energy (kWh)")
-    ax1.set_title("Community Energy Consumption: Grid Import vs. P2P Buy")
-    ax1.legend()
-    ax1.grid(True, linestyle='--', alpha=0.6)
-
-    # Subplot 2: Grid Export vs P2P Sell
-    ax2.plot(step_group["global_step"], step_group["grid_export"], label="Grid Export", color='orange')
-    ax2.plot(step_group["global_step"], step_group["p2p_sell"], label="P2P Sell", color='b')
-    ax2.set_xlabel("Global Timestep")
-    ax2.set_ylabel("Energy (kWh)")
-    ax2.set_title("Community Energy Generation: Grid Export vs. P2P Sell")
-    ax2.legend()
-    ax2.grid(True, linestyle='--', alpha=0.6)
-     
-    plt.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "combined_energy_flows_timeseries.png"))
-    plt.close()
-
-    # Stacked Bar of Daily Energy Sources
-    daily_agg = all_days_df.groupby("day").sum(numeric_only=True)
-     
-    plt.figure(figsize=(12, 7))
-    plt.bar(daily_agg.index, daily_agg["grid_import_with_p2p"], label="Grid Import (with P2P)", color='crimson')
-    plt.bar(daily_agg.index, daily_agg["p2p_buy"], bottom=daily_agg["grid_import_with_p2p"], label="P2P Buy", color='limegreen')
-    plt.plot(daily_agg.index, daily_agg["grid_import_no_p2p"], label="Baseline Grid Import (No P2P)", color='blue', linestyle='--', marker='o')
-     
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Daily Energy Procurement: Baseline vs. P2P+Grid")
-    plt.xticks(daily_agg.index)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_energy_procurement_stacked.png"))
-    plt.close()
-
-    # Fairness Metrics Over Time
-    plt.figure(figsize=(12, 6))
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_cost_savings"], label="Cost Savings Fairness", marker='o')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_grid_reduction"], label="Grid Reduction Fairness", marker='s')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_reward"], label="Reward Fairness", marker='^')
-    plt.xlabel("Day")
-    plt.ylabel("Jain's Fairness Index")
-    plt.title("Daily Fairness Metrics")
-    plt.xticks(plot_daily_df["day"])
-    plt.ylim(0, 1.05)
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_fairness_metrics.png"))
-    plt.close()
-
-    # Per-House Savings and Reductions
-    fig, ax1 = plt.subplots(figsize=(15, 7))
-     
-    house_ids_str = house_level_df.index.astype(str)
-    bar_width = 0.4
-    index = np.arange(len(house_ids_str))
-
-    # Bar chart for cost savings
-    color1 = 'tab:green'
-    ax1.set_xlabel('House ID')
-    ax1.set_ylabel('Total Cost Savings ($)', color=color1)
-    ax1.bar(index - bar_width/2, house_level_df["cost_savings"], bar_width, label='Cost Savings', color=color1)
-    ax1.tick_params(axis='y', labelcolor=color1)
-    ax1.set_xticks(index)
-    ax1.set_xticklabels(house_ids_str, rotation=45, ha="right")
-     
-    # Second y-axis for grid import reduction
-    ax2 = ax1.twinx()
-    color2 = 'tab:blue'
-    ax2.set_ylabel('Total Grid Import Reduction (kWh)', color=color2)
-    ax2.bar(index + bar_width/2, house_level_df["import_reduction"], bar_width, label='Import Reduction', color=color2)
-    ax2.tick_params(axis='y', labelcolor=color2)
-
-    plt.title(f'Total Cost Savings & Grid Import Reduction Per House (over {days_to_evaluate} days)')
-    fig.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "per_house_summary.png"))
-    plt.close()
-     
-    # Price Dynamics for a Single Day
-    day1_prices = all_days_df[all_days_df['day'] == 1][['step', 'grid_price', 'peer_price']].drop_duplicates()
-    plt.figure(figsize=(12, 6))
-    plt.plot(day1_prices['step'], day1_prices['grid_price'], label='Grid Price', color='darkorange')
-    plt.plot(day1_prices['step'], day1_prices['peer_price'], label='P2P Price', color='teal')
-    plt.xlabel("Timestep of Day")
-    plt.ylabel("Price ($/kWh)")
-    plt.title("Price Dynamics on Day 1")
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.6)
-    plt.savefig(os.path.join(plots_dir, "price_dynamics_day1.png"))
-    plt.close()
-     
-    # Battery State of Charge for Sample Houses
-    day1_df = all_days_df[all_days_df['day'] == 1]
-    battery_houses = day1_df.dropna(subset=['soc'])['house'].unique()
-     
-    if len(battery_houses) > 0:
-        sample_houses = battery_houses[:min(4, len(battery_houses))]
-        plt.figure(figsize=(12, 6))
-        for house in sample_houses:
-            house_df = day1_df[day1_df['house'] == house]
-            plt.plot(house_df['step'], house_df['soc'] * 100, label=f'House {house}')
-         
-        plt.xlabel("Timestep of Day")
-        plt.ylabel("State of Charge (%)")
-        plt.title("Battery SoC on Day 1 for Sample Houses")
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.6)
-        plt.savefig(os.path.join(plots_dir, "soc_dynamics_day1.png"))
-        plt.close()
-
-    print("All plots have been generated and saved. Evaluation complete.")
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/mappo/mappo_train.py

@@ -1,439 +0,0 @@
-import os
-import sys
-import re             # ← add thist
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-import pandas as pd
-import time
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sharer_battery_env import SolarSharer
-from mappo.trainer.mappo import MAPPO
- 
-def main():
-
-    STATE_TO_RUN = "pennsylvania" # "pennsylvania" or "colorado" or "oklahoma"
-
-    # --- Set the path to your training data ---
-    DATA_FILE_PATH = "/Users/ananygupta/Desktop/Final_revision/Australia_data/processed_data_ausgrid_100_houses.csv"
-    num_episodes = 10000
-    # total # of episodes you want to run
-    batch_size = 256    # e.g. 512, 1024, 2048
-    checkpoint_interval = 100000
-    window_size = 32        # ← group episodes in blocks of 30
-
-
-    env = SolarSharer(
-        data_path=DATA_FILE_PATH,
-        state=STATE_TO_RUN,
-        time_freq="30T"
-    )
-    ############################################################################################
-    # ─── Sanity check: env I/O shapes ─────────────────────────────────────
-    print("Observation space:", env.observation_space)
-    print("Action space     :", env.action_space)
-
-    # Reset and inspect obs
-    obs = env.reset()
-    print(f"Reset returned {len(obs)} agent observations; each obs shape: {np.array(obs).shape}")
-
-    # Sample random actions and do one step
-    dummy_actions = np.random.rand(env.num_agents, env.action_space.shape[1]).astype(np.float32)
-    next_obs, rewards, done, info = env.step(dummy_actions)
-    print(f"Step outputs → next_obs: {len(next_obs)}×{np.array(next_obs).shape[1]}, "
-          f"rewards: {len(rewards)}, done: {done}")
-    print("Info keys:", list(info.keys()))
-    # ────────────────────────────────────────────────────────────────
-
-    # Count the number of houses in each group
-    env.group_counts = {
-        0: env.agent_groups.count(0),
-        1: env.agent_groups.count(1)
-    }
-    print(f"Number of houses in each group: {env.group_counts}")
-
-    max_steps = env.num_steps
-
-    # dims from the env
-    num_agents = env.num_agents
-    local_state_dim = env.observation_space.shape[1]
-    action_dim = env.action_space.shape[1]
-
-    # ─── Build a unique run directory ───────────────────────────
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"mappo_{STATE_TO_RUN}_{num_agents}agents_{num_episodes}eps_{timestamp}"
-    root_dir  = os.path.join("Testing_with_australia_data", run_name)
-    os.makedirs(root_dir, exist_ok=True)
-    print(f"Saving training outputs to: {root_dir}")
-
-    logs_dir  = os.path.join(root_dir, "logs")
-    plots_dir = os.path.join(root_dir, "plots")
-    os.makedirs(logs_dir, exist_ok=True)
-    os.makedirs(plots_dir, exist_ok=True)
-
-
-    # Create the MeanField agent
-    mappo = MAPPO(
-        n_agents=num_agents,
-        local_dim=local_state_dim,
-        global_dim=num_agents * local_state_dim,
-        act_dim=action_dim,
-        lr=2e-4,
-        gamma=0.95,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=4,
-        batch_size=batch_size
-    )
-
-
-    # ─────────────── Tracking / Logging Variables ───────────────
-    episode_rewards = []             # mean reward per episode (averaged across agents)
-    episode_total_rewards = []       # total reward per episode (sum across agents)
-    block_mean_rewards = []          # mean of mean-episode-rewards for each block of window_size
-    block_total_rewards = []         # sum of total-episode-rewards for each block of window_size
-
-    agent_rewards_log = [[] for _ in range(num_agents)]
-    best_mean_reward = -1e9
-    best_model_path = os.path.join(logs_dir, "best_model.pth")
-
-
-    daily_rewards = []               # alias for episode_rewards
-    monthly_rewards = []             # just kept in case you want the old logic
-
-    training_start_time = time.time()
-    episode_durations = []
-    total_steps_global = 0
-    episode_log_data = []
-    # ADD THIS LINE to store the new metrics from the environment
-    performance_metrics_log = [] # This will hold the detailed performance data for each episode.
-
-
-    agent_charge_log = [[] for _ in range(num_agents)]  # Track charge actions
-    agent_discharge_log = [[] for _ in range(num_agents)]  # Track discharge actions
-
-
-    # ──────────── Training Loop ────────────
-    for episode in range(1, num_episodes + 1):
-        episode_start_time = time.time()
-
-        obs = np.array(env.reset(), dtype=np.float32)
-
-
-        # ADD THIS BLOCK to collect metrics from the *previous* episode
-        # =================================================================
-        # The env.reset() call above finalized the metrics for the episode that just finished.
-        # We retrieve them here. We check `if episode > 1` because there are no
-        # metrics to collect before the first episode has run.
-        if episode > 1:
-            # Call the getter method you added to the environment
-            last_episode_metrics = env.get_episode_metrics()
-
-            # Add the corresponding episode number for merging later
-            last_episode_metrics['Episode'] = episode - 1
-
-            # Append the dictionary of metrics to our new log
-            performance_metrics_log.append(last_episode_metrics)
-        # =================================================================
-
-        total_reward = np.zeros(num_agents, dtype=np.float32)
-        done = False
-        step_count = 0
-        day_logs = []
-        episode_charges = [[] for _ in range(num_agents)]
-        episode_discharges = [[] for _ in range(num_agents)]
-
-        while not done:
-
-            # flatten the joint state once per step
-            # build global state and pick actions
-            # obs is already a NumPy array of shape (num_agents, local_dim)
-            global_obs      = obs.flatten()
-            actions, logps  = mappo.select_action(obs, global_obs)
-
-            # step environment
-            next_obs_list, rewards, done, info = env.step(actions)
-
-            # convert next observations to NumPy array too
-            next_obs        = np.array(next_obs_list, dtype=np.float32)
-            next_global_obs = next_obs.flatten()
-
-
-            # store transition
-            # ensure fast conversion to torch.Tensor
-            local_obs_arr      = np.array(obs,      dtype=np.float32)
-
-            mappo.store(
-                local_obs_arr,
-                global_obs,
-                actions,
-                logps,
-                rewards,
-                done,
-                next_global_obs
-            )
-            total_reward += rewards
-            obs = next_obs
-            step_count += 1
-            total_steps_global += 1
-
-            day_logs.append({
-                "step": step_count - 1,
-                "grid_import_no_p2p": info["grid_import_no_p2p"],
-                "grid_import_with_p2p": info["grid_import_with_p2p"],
-                "p2p_buy": info["p2p_buy"],
-                "p2p_sell": info["p2p_sell"],
-                "costs": info["costs"],  # Capture costs for analysis
-                "charge_amount": info.get("charge_amount", np.zeros(num_agents)),  # New
-                "discharge_amount": info.get("discharge_amount", np.zeros(num_agents))  # New
-            })
-
-            if step_count >= max_steps:
-                break
-
-        # ─── After each episode ───
-        # 1) Compute per-episode metrics
-        sum_ep_reward = float(np.sum(total_reward))        # total reward across all agents for this episode
-        mean_ep_reward = float(np.mean(total_reward))      # mean reward across agents for this episode
-
-        episode_total_rewards.append(sum_ep_reward)
-        episode_rewards.append(mean_ep_reward)
-        daily_rewards.append(mean_ep_reward)
-
-        # 2) If we just finished a block of window_size episodes, aggregate
-        if len(daily_rewards) % window_size == 0:
-            # Sum of total rewards over the last window_size episodes
-            last_totals = episode_total_rewards[-window_size:]
-            block_sum = sum(last_totals)
-            block_total_rewards.append(block_sum)
-
-            # Mean of mean-episode-rewards over the last window_size episodes
-            last_means = daily_rewards[-window_size:]
-            block_mean = sum(last_means) / window_size
-            block_mean_rewards.append(block_mean)
-
-            block_idx = len(block_mean_rewards)
-            print(
-                f"→ Completed Block {block_idx} "
-                f"| Episodes { (block_idx-1)*window_size + 1 }–{ block_idx*window_size } "
-                f"| Block Total Reward: {block_sum:.3f} "
-                f"| Block Mean Reward: {block_mean:.3f}"
-            )
-
-        # 3) Log agent-level rewards
-        for i in range(num_agents):
-            agent_rewards_log[i].append(total_reward[i])
-            episode_charges[i].append(actions[i][4])
-            episode_discharges[i].append(actions[i][5])
-
-        # 4) Summarize P2P steps (unchanged from your original code)
-        steps_data = []
-        for entry in day_logs:
-            step_idx = entry["step"]
-            p2p_buy_array  = entry["p2p_buy"]
-            p2p_sell_array = entry["p2p_sell"]
-            grid_no_p2p_array   = entry["grid_import_no_p2p"]
-            grid_with_p2p_array = entry["grid_import_with_p2p"]
-
-            steps_data.append({
-                "step": step_idx,
-                "p2p_buy_sum":  float(np.sum(p2p_buy_array)),
-                "p2p_sell_sum": float(np.sum(p2p_sell_array)),
-                "grid_import_no_p2p_sum":   float(np.sum(grid_no_p2p_array)),
-                "grid_import_with_p2p_sum": float(np.sum(grid_with_p2p_array))
-            })
-
-
-        baseline_cost = np.sum([np.sum(entry["grid_import_no_p2p"]) * env.get_grid_price(entry["step"])
-                                for entry in day_logs])
-        actual_cost = np.sum([np.sum(entry["costs"]) for entry in day_logs])
-        cost_reduction = (baseline_cost - actual_cost) / baseline_cost
-
-        # at end of episode
-        mappo.update()  # Update the MAPPO agent
-
-
-        # save if best
-        if mean_ep_reward > best_mean_reward:
-            best_mean_reward = mean_ep_reward
-            mappo.save(best_model_path)
-
-        if episode % checkpoint_interval == 0:
-            ckpt_path = os.path.join(logs_dir, f"checkpoint_{episode}.pth")
-            mappo.save(ckpt_path)
-        # CORRECTED TIMING AND LOGGING
-        episode_end_time = time.time()
-        episode_duration = episode_end_time - episode_start_time
-
-        # Move the print statement here
-        print(
-            f"Episode {episode}/{num_episodes} "
-            f"| Time per Episode: {episode_duration:.2f}s "
-            f"| Steps: {step_count} "
-            f"| Mean Reward: {mean_ep_reward:.3f} "
-            f"| Cost Reduction: {cost_reduction:.2%}"
-        )
-
-        # Record data in our per-episode log
-        episode_log_data.append({
-            "Episode": episode,
-            "Steps": step_count,
-            "Mean_Reward": mean_ep_reward,
-            "Total_Reward": sum_ep_reward,
-            "Cost_Reduction_Pct": cost_reduction * 100,  # New
-            "Baseline_Cost": baseline_cost,  # New
-            "Actual_Cost": actual_cost,  # New
-            "Episode_Duration": episode_duration,
-            "Total_Charge": np.sum([np.sum(entry["charge_amount"]) for entry in day_logs]),  # New
-            "Total_Discharge": np.sum([np.sum(entry["discharge_amount"]) for entry in day_logs])  # New
-        })
-    for i in range(num_agents):
-        agent_charge_log[i].append(np.mean(episode_charges[i]))
-        agent_discharge_log[i].append(np.mean(episode_discharges[i]))
-
-    # ADD THIS BLOCK TO CAPTURE THE FINAL EPISODE'S METRICS
-    # =================================================================
-    # After the loop, the metrics for the final episode (num_episodes) are ready.
-    # We collect them here to ensure the log is complete.
-    final_episode_metrics = env.get_episode_metrics()
-    final_episode_metrics['Episode'] = num_episodes
-    performance_metrics_log.append(final_episode_metrics)
-    # =================================================================
-
-
-
-    # ─── End of all training ───
-    training_end_time = time.time()
-    total_training_time = training_end_time - training_start_time
-
-    # Save out per-episode agent rewards + mean rewards
-    np.save(os.path.join(logs_dir, "agent_rewards.npy"), np.array(agent_rewards_log))
-    np.save(os.path.join(logs_dir, "mean_rewards.npy"),  np.array(episode_rewards))
-    np.save(os.path.join(logs_dir, "total_rewards.npy"), np.array(episode_total_rewards))
-
-    ################################# PLOTTING & LOGGING ##################################################################
-    # ─────────── Create Final DataFrame for Logging and Plotting ───────────
-
-    # 1. Create a DataFrame from the original log data (rewards, costs, etc.)
-    df_rewards_log = pd.DataFrame(episode_log_data)
-
-    # 2. Create a DataFrame from the new performance metrics log
-    df_perf_log = pd.DataFrame(performance_metrics_log)
-
-    # 3. Merge the two DataFrames on the 'Episode' column.
-    #    This combines all metrics into a single table.
-    df_final_log = pd.merge(df_rewards_log, df_perf_log.drop(columns=[
-        'degradation_cost_over_time',
-        'cost_savings_over_time',
-        'grid_reduction_over_time'
-    ]), on="Episode")
-
-
-    # ─────────── PLOTTING ───────────
-
-    # Ensure plot directory exists
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Helper: centered moving average
-    def moving_avg(series, window):
-        return pd.Series(series).rolling(window=window, center=True, min_periods=1).mean().to_numpy()
-
-    # Smoothing window (in episodes)
-    ma_window = 300
-    episodes = np.arange(1, num_episodes + 1)
-
-    # 1. Mean Reward moving average
-    reward_ma = moving_avg(df_final_log["Mean_Reward"], ma_window)
-    plt.figure(figsize=(8,5))
-    plt.plot(episodes, reward_ma, linewidth=2, label=f"Mean Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Mean Reward")
-    plt.title("MAPPO: Mean Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "mean_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # 2. Total Reward moving average
-    total_ma = moving_avg(df_final_log["Total_Reward"], ma_window)
-    plt.figure(figsize=(8,5))
-    plt.plot(episodes, total_ma, linewidth=2, label=f"Total Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Total Reward")
-    plt.title("MAPPO: Total Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "total_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # 3. Cost Reduction (%) moving average
-    cost_ma = moving_avg(df_final_log["Cost_Reduction_Pct"], ma_window)
-    plt.figure(figsize=(8,5))
-    plt.plot(episodes, cost_ma, linewidth=2, label="Cost Reduction MA (%)")
-    plt.xlabel("Episode")
-    plt.ylabel("Cost Reduction (%)")
-    plt.title("MAPPO: Cost Reduction Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "cost_reduction_ma.png"), dpi=200)
-    plt.close()
-
-    # 4. Battery Degradation Cost moving average
-    degradation_ma = moving_avg(df_final_log["battery_degradation_cost_total"], ma_window)
-    plt.figure(figsize=(8,5))
-    plt.plot(episodes, degradation_ma, linewidth=2, label=f"Degradation Cost MA (win={ma_window})", color='purple')
-    plt.xlabel("Episode")
-    plt.ylabel("Total Degradation Cost ($)")
-    plt.title("MAPPO: Battery Degradation Cost Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "degradation_cost_ma.png"), dpi=200)
-    plt.close()
-
-
-    # Final confirmation message
-    print(f"\nAll moving-average plots saved to: {plots_dir}")
-
-
-    # ─── Save Final Logs to CSV ───
-
-    # 1. Add the total training time as a new row to the DataFrame
-    total_time_row = pd.DataFrame([{
-        "Episode": "Total_Training_Time",
-        "Episode_Duration": total_training_time
-    }])
-    df_to_save = pd.concat([df_final_log, total_time_row], ignore_index=True)
-
-
-    # 2. Define the path for the final CSV file.
-    log_csv_path = os.path.join(logs_dir, "training_performance_log.csv")
-
-    # 3. Select and reorder columns for the final CSV
-    columns_to_save = [
-        "Episode",
-        "Mean_Reward",
-        "Total_Reward",
-        "Cost_Reduction_Pct",
-        "Episode_Duration",
-        "battery_degradation_cost_total",
-    ]
-    df_to_save = df_to_save[columns_to_save]
-
-
-    # 4. Save the comprehensive DataFrame to CSV.
-    df_to_save.to_csv(log_csv_path, index=False)
-
-    print(f"Saved comprehensive training performance log to: {log_csv_path}")
-
-    # ─── Final Timings Printout ───
-    print("\n" + "="*50)
-    print("TRAINING COMPLETE".center(50))
-    print(f"Total training time: {total_training_time:.2f} seconds")
-    print("="*50)
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/mappo/trainer/mappo.py

@@ -1,243 +0,0 @@
-# mappo.py
-import torch
-import torch.nn as nn
-import random
-import numpy as np
-from torch.distributions import Normal
-
-
-def set_global_seed(seed: int):
-    random.seed(seed)                # Python
-    np.random.seed(seed)             # NumPy
-    torch.manual_seed(seed)          # PyTorch CPU
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)  # PyTorch GPU
-    # make CuDNN deterministic (may slow you down a bit):
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark     = False
-
-
-# Universal device selection
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-    print("Using CUDA (NVIDIA GPU)")
-# elif torch.backends.mps.is_available():
-#     device = torch.device("mps")
-#     print("Using MPS (Apple Silicon GPU)")
-else:
-    device = torch.device("cpu")
-    print("Using CPU")
-
-# fix EVERYTHING
-SEED = 42
-set_global_seed(SEED)
-
-
-class MLP(nn.Module):
-    def __init__(self, input_dim, hidden_dims, output_dim):
-        super().__init__()
-        layers = []
-        last_dim = input_dim
-        for h in hidden_dims:
-            layers += [nn.Linear(last_dim, h), nn.ReLU()]
-            last_dim = h
-        layers.append(nn.Linear(last_dim, output_dim))
-        self.net = nn.Sequential(*layers)
-
-    def forward(self, x):
-        return self.net(x)
-
-class Actor(nn.Module):
-    def __init__(self, obs_dim, act_dim, hidden=(64,64)):
-        super().__init__()
-        self.net = MLP(obs_dim, hidden, act_dim)
-        self.log_std = nn.Parameter(torch.zeros(act_dim))
-
-    def forward(self, x):
-        mean = self.net(x)
-        std = torch.exp(self.log_std)
-        return mean, std
-
-class Critic(nn.Module):
-    def __init__(self, state_dim, hidden=(128,128)):
-        super().__init__()
-        self.net = MLP(state_dim, hidden, 1)
-
-    def forward(self, x):
-        return self.net(x).squeeze(-1)
-
-class MAPPO:
-    def __init__(
-        self,
-        n_agents,
-        local_dim,
-        global_dim,
-        act_dim,
-        lr=3e-4,
-        gamma=0.99,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=10,
-        batch_size=1024
-    ):
-        self.n_agents = n_agents
-        self.gamma    = gamma
-        self.lam      = lam
-        self.clip_eps = clip_eps
-        self.k_epochs = k_epochs
-        self.batch_size = batch_size
-
-        self.actor  = Actor(local_dim, act_dim).to(device)
-        self.critic = Critic(global_dim).to(device)
-
-        self.opt_a = torch.optim.Adam(self.actor.parameters(), lr=lr)
-        self.opt_c = torch.optim.Adam(self.critic.parameters(), lr=lr)
-            
-        self.local_dim  = local_dim
-        self.global_dim = global_dim
-        self.act_dim    = act_dim
-
-        self.clear_buffer()
-
-    def clear_buffer(self):
-        self.ls     = []  # local observations
-        self.gs     = []  # global observations
-        self.ac     = []  # actions
-        self.lp     = []  # log-probs
-        self.rw     = []  # rewards
-        self.done   = []  # done flags
-        self.next_gs = [] # next global observations
-
-    @torch.no_grad()
-    def select_action(self, local_obs, global_obs):
-        l = torch.FloatTensor(local_obs).to(device)
-        mean, std = self.actor(l)
-        dist = Normal(mean, std)
-        a = dist.sample()
-        return a.cpu().numpy(), dist.log_prob(a).sum(-1).cpu().numpy()
-
-    def store(self, local_obs, global_obs, action, logp, reward, done, next_global_obs):
-        self.ls.append(local_obs)
-        self.gs.append(global_obs)
-        self.ac.append(action)
-        self.lp.append(logp)
-        self.rw.append(reward)
-        self.done.append(done)
-        self.next_gs.append(next_global_obs)
-
-    def compute_gae(self, values):
-        """
-        values: torch.Tensor shape [T] (one central V(s) per timestep)
-        returns:
-        adv_flat: torch.Tensor shape [T * n_agents]
-        ret_flat: torch.Tensor shape [T * n_agents]
-        """
-        # 1) get raw arrays
-        vals_1d = values.cpu().numpy()         # [T]
-        T = len(vals_1d)
-        N = self.n_agents
-
-        # 2) broadcast to per-agent
-        #    vals_agent[t,i] = V(state_t)
-        vals_agent = np.tile(vals_1d[:,None], (1, N))  # [T,N]
-
-        # 3) build next_vals likewise
-        next_vals = np.zeros_like(vals_agent)          # [T,N]
-        next_vals[:-1] = vals_agent[1:]
-        # if episode didn’t end at final step, bootstrap last:
-        if not self.done[-1]:
-            with torch.no_grad():
-                v_last = self.critic(
-                    torch.FloatTensor(self.next_gs[-1]).to(device)
-                ).cpu().item()
-            next_vals[-1, :] = v_last
-
-        # 4) GAE loop over (T,N)
-        adv = np.zeros_like(vals_agent, dtype=np.float32)
-        prev_adv = np.zeros(N, dtype=np.float32)
-        for t in reversed(range(T)):
-            mask = 1.0 - float(self.done[t])        # scalar 0/1
-            rew_t = np.array(self.rw[t], dtype=np.float32)  # [N]
-            delta = rew_t + self.gamma * next_vals[t] * mask - vals_agent[t]
-            prev_adv = delta + self.gamma * self.lam * mask * prev_adv
-            adv[t] = prev_adv
-
-        # 5) compute returns & flatten
-        ret = adv + vals_agent                        # [T,N]
-        adv_flat = torch.from_numpy(adv.flatten()).to(device)
-        ret_flat = torch.from_numpy(ret.flatten()).to(device)
-        return adv_flat, ret_flat
-    
-
-    def update(self):
-        # 1) Raw global states tensor [T, G]
-        raw_gs = torch.FloatTensor(self.gs).to(device)  # [T, G]
-
-        # 2) Compute one value V(s_t) per timestep
-        with torch.no_grad():
-            vals = self.critic(raw_gs).cpu()  # [T]
-
-        # 3) Compute advantages and returns using GAE (returns flattened [T*N])
-        adv_flat, ret_flat = self.compute_gae(vals)  # both shape [T * N]
-
-        # 4) Prepare per-agent flattened training tensors
-        # Local states [T*N, local_dim]
-        ls = torch.FloatTensor(self.ls).view(-1, self.local_dim).to(device)
-        # Actions [T*N, act_dim]
-        ac = torch.FloatTensor(self.ac).view(-1, self.act_dim).to(device)
-        # Old log-probs [T*N]
-        old_lp = torch.FloatTensor(self.lp).view(-1).to(device)
-
-        # Broadcast global states to per-agent: [T, G] -> [T, N, G] -> [T*N, G]
-        gs = raw_gs.unsqueeze(1).expand(-1, self.n_agents, -1)  # [T, N, G]
-        gs = gs.reshape(-1, self.global_dim).to(device)          # [T*N, G]
-
-        # Create dataset and loader
-        dataset = torch.utils.data.TensorDataset(
-            ls, gs, ac, old_lp, adv_flat, ret_flat
-        )
-        gen = torch.Generator()
-        gen.manual_seed(SEED)
-        loader = torch.utils.data.DataLoader(
-            dataset,
-            batch_size=self.batch_size,
-            shuffle=True,
-            num_workers=0,
-            generator=gen
-        )
-        # 5) PPO update loop
-        for _ in range(self.k_epochs):
-            for b_ls, b_gs, b_ac, b_lp, b_adv, b_ret in loader:
-                # Actor update
-                mean, std = self.actor(b_ls)
-                dist = Normal(mean, std)
-                lp_new = dist.log_prob(b_ac).sum(-1)
-                ratio = torch.exp(lp_new - b_lp)
-                surr1 = ratio * b_adv
-                surr2 = torch.clamp(ratio, 1 - self.clip_eps, 1 + self.clip_eps) * b_adv
-                actor_loss = -torch.min(surr1, surr2).mean()
-
-                self.opt_a.zero_grad()
-                actor_loss.backward()
-                self.opt_a.step()
-
-                # Critic update
-                val_pred = self.critic(b_gs)
-                critic_loss = nn.MSELoss()(val_pred, b_ret)
-
-                self.opt_c.zero_grad()
-                critic_loss.backward()
-                self.opt_c.step()
-
-        # 6) Clear buffers for next rollout
-        self.clear_buffer()
-
-
-    def save(self, path):
-        torch.save({'actor': self.actor.state_dict(),
-                    'critic': self.critic.state_dict()}, path)
-
-    def load(self, path):
-        data = torch.load(path, map_location=device)
-        self.actor.load_state_dict(data['actor'])
-        self.critic.load_state_dict(data['critic'])

Other_algorithms/Flat_System/meanfield/meanfield_evaluation.py

@@ -1,492 +0,0 @@
-#mfac_evaluate.py
-import os
-import sys
-import time
-import re
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-import torch
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from meanfield.trainer.mfac import MeanField
-
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-def compute_jains_fairness(values: np.ndarray) -> float:
-    if len(values) == 0:
-        return 0.0
-    if np.all(values == 0):
-        return 1.0
-    num = (values.sum())**2
-    den = len(values) * (values**2).sum()
-    return num / den
-
-def main():
-    # User parameters
-    MODEL_PATH = "/path/to/project/meanfield_pennsylvania_100agents_10000eps/logs/best_model.pth"
-    DATA_PATH = "/path/to/project/testing/100houses_30days_TEST.csv"
-    DAYS_TO_EVALUATE = 30
-
-    model_path = MODEL_PATH
-    data_path = DATA_PATH
-    days_to_evaluate = DAYS_TO_EVALUATE
-    SOLAR_THRESHOLD = 0.1
-
-    # Env setup
-    env = SolarSys(
-        data_path=data_path,
-        state="pennsylvania",
-        time_freq="3H"
-    )
-    eval_steps = env.num_steps
-    house_ids = env.house_ids
-    num_agents = env.num_agents
-
-    # Generate a unique eval run folder
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"eval_mappo_{num_agents}agents_{days_to_evaluate}days_{timestamp}"
-    output_folder = os.path.join("runs_with_battery", run_name)
-    logs_dir = os.path.join(output_folder, "logs")
-    plots_dir = os.path.join(output_folder, "plots")
-    for d in (logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Saving evaluation outputs to: {output_folder}")
-        
-    local_dim = env.observation_space.shape[1]
-    global_dim = num_agents * local_dim
-    act_dim = env.action_space.shape[1]
-    
-    mfac = MeanField(
-        n_agents=num_agents,
-        local_dim=local_dim,
-        global_dim=global_dim,
-        act_dim=act_dim,
-        lr=2e-4,
-        gamma=0.95,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=10,
-        batch_size=1024
-    )
-
-    # Loadmfac checkpoint
-    mfac.load(model_path)
-    mfac.actor.to(device).eval()
-    mfac.critic.to(device).eval()
-
-    # Prepare logs
-    all_logs = []
-    daily_summaries = []
-    step_timing_list = []
-     
-    evaluation_start = time.time()
-
-    for day_idx in range(days_to_evaluate):
-        obs = env.reset()
-        obs = np.array(obs, dtype=np.float32)
-        done = False
-        step_count = 0
-        day_logs = []
-
-        while not done:
-            step_start_time = time.time()
-            global_obs = np.array(obs).flatten()
-
-            # Select actions withmfac
-            actions, _ =mfac.select_action(obs, global_obs)
-
-            next_obs, rewards, done, info = env.step(actions)
-            next_obs = np.array(next_obs, dtype=np.float32)
-
-            # Consolidated Logging
-            step_end_time = time.time()
-            step_duration = step_end_time - step_start_time
-             
-            print(f"[Day {day_idx+1}, Step {step_count}] Step time: {step_duration:.6f} seconds")
-
-            step_timing_list.append({
-                "day": day_idx + 1,
-                "step": step_count,
-                "step_time_s": step_duration
-            })
-
-            grid_price_now = env.get_grid_price(step_count)
-            peer_price_now = info.get("peer_price", env.get_peer_price(step_count,
-                                float(info["p2p_sell"].sum()),
-                                float(info["p2p_buy"].sum())))
-
-            for i, hid in enumerate(house_ids):
-                is_battery_house = hid in env.batteries
-                p2p_buy = float(info["p2p_buy"][i])
-                p2p_sell = float(info["p2p_sell"][i])
-                charge_amount = float(info.get("charge_amount", [0]*num_agents)[i])
-                discharge_amount = float(info.get("discharge_amount", [0]*num_agents)[i])
-
-                day_logs.append({
-                    "day": day_idx + 1,
-                    "step": step_count,
-                    "house": hid,
-                    "grid_import_no_p2p": float(info["grid_import_no_p2p"][i]),
-                    "grid_import_with_p2p": float(info["grid_import_with_p2p"][i]),
-                    "grid_export": float(info.get("grid_export", [0]*num_agents)[i]),
-                    "p2p_buy": p2p_buy,
-                    "p2p_sell": p2p_sell,
-                    "actual_cost": float(info["costs"][i]),
-                    "baseline_cost": float(info["grid_import_no_p2p"][i]) * grid_price_now,
-                    "total_demand": float(env.demands[hid][step_count]),
-                    "total_solar": float(env.solars[hid][step_count]),
-                    "grid_price": grid_price_now,
-                    "peer_price": peer_price_now,
-                    "soc": (env.batteries[hid]["soc"] / env.batteries[hid]["max_capacity"]) if is_battery_house else np.nan,
-                    "degradation_cost": ((charge_amount + discharge_amount) * env.batteries[hid]["degradation_cost_per_kwh"]) if is_battery_house else 0.0,
-                    "reward": float(rewards[i]),
-                })
-
-            obs = next_obs
-            step_count += 1
-            if step_count >= eval_steps:
-                break
-         
-        day_df = pd.DataFrame(day_logs)
-        all_logs.extend(day_logs)
-
-        # Consolidated daily summary calculation
-        grouped_house = day_df.groupby("house").sum(numeric_only=True)
-        grouped_step = day_df.groupby("step").sum(numeric_only=True)
-
-        total_demand = grouped_step["total_demand"].sum()
-        total_solar = grouped_step["total_solar"].sum()
-        total_p2p_buy = grouped_house["p2p_buy"].sum()
-        total_p2p_sell = grouped_house["p2p_sell"].sum()
-
-        baseline_cost_per_house = grouped_house["baseline_cost"]
-        actual_cost_per_house = grouped_house["actual_cost"]
-        cost_savings_per_house = baseline_cost_per_house - actual_cost_per_house
-        day_total_cost_savings = cost_savings_per_house.sum()
-         
-        if baseline_cost_per_house.sum() > 0:
-            overall_cost_savings_pct = day_total_cost_savings / baseline_cost_per_house.sum()
-        else:
-            overall_cost_savings_pct = 0.0
-
-        baseline_import_per_house = grouped_house["grid_import_no_p2p"]
-        actual_import_per_house = grouped_house["grid_import_with_p2p"]
-        import_reduction_per_house = baseline_import_per_house - actual_import_per_house
-        day_total_import_reduction = import_reduction_per_house.sum()
-         
-        if baseline_import_per_house.sum() > 0:
-            overall_import_reduction_pct = day_total_import_reduction / baseline_import_per_house.sum()
-        else:
-            overall_import_reduction_pct = 0.0
-
-        fairness_cost_savings = compute_jains_fairness(cost_savings_per_house.values)
-        fairness_import_reduction = compute_jains_fairness(import_reduction_per_house.values)
-        fairness_rewards = compute_jains_fairness(grouped_house["reward"].values)
-        fairness_p2p_buy = compute_jains_fairness(grouped_house["p2p_buy"].values)
-        fairness_p2p_sell = compute_jains_fairness(grouped_house["p2p_sell"].values)
-        fairness_p2p_total = compute_jains_fairness((grouped_house["p2p_buy"] + grouped_house["p2p_sell"]).values)
-        day_total_degradation_cost = grouped_house["degradation_cost"].sum()
-
-        daily_summaries.append({
-            "day": day_idx + 1,
-            "day_total_demand": total_demand,
-            "day_total_solar": total_solar,
-            "day_p2p_buy": total_p2p_buy,
-            "day_p2p_sell": total_p2p_sell,
-            "cost_savings_abs": day_total_cost_savings,
-            "cost_savings_pct": overall_cost_savings_pct,
-            "fairness_cost_savings": fairness_cost_savings,
-            "grid_reduction_abs": day_total_import_reduction,
-            "grid_reduction_pct": overall_import_reduction_pct,
-            "fairness_grid_reduction": fairness_import_reduction,
-            "fairness_reward": fairness_rewards,
-            "fairness_p2p_buy": fairness_p2p_buy,
-            "fairness_p2p_sell": fairness_p2p_sell,
-            "fairness_p2p_total": fairness_p2p_total,
-            "total_degradation_cost": day_total_degradation_cost
-        })
-
-    # Final processing and saving
-    evaluation_end = time.time()
-    total_eval_time = evaluation_end - evaluation_start
-    print(f"\nEvaluation loop finished. Total time: {total_eval_time:.2f} seconds.")
-
-    all_days_df = pd.DataFrame(all_logs)
-    combined_csv_path = os.path.join(logs_dir, "step_logs_all_days.csv")
-    all_days_df.to_csv(combined_csv_path, index=False)
-    print(f"Saved combined step-level logs to: {combined_csv_path}")
-
-    step_timing_df = pd.DataFrame(step_timing_list)
-    timing_csv_path = os.path.join(logs_dir, "step_timing_log.csv")
-    step_timing_df.to_csv(timing_csv_path, index=False)
-    print(f"Saved step timing logs to: {timing_csv_path}")
-
-    house_level_df = all_days_df.groupby("house").agg({
-        "baseline_cost": "sum",
-        "actual_cost": "sum",
-        "grid_import_no_p2p": "sum",
-        "grid_import_with_p2p": "sum",
-        "degradation_cost": "sum"
-    })
-    house_level_df["cost_savings"] = house_level_df["baseline_cost"] - house_level_df["actual_cost"]
-    house_level_df["import_reduction"] = house_level_df["grid_import_no_p2p"] - house_level_df["grid_import_with_p2p"]
-     
-    house_summary_csv = os.path.join(logs_dir, "summary_per_house.csv")
-    house_level_df.to_csv(house_summary_csv)
-    print(f"Saved final summary per house to: {house_summary_csv}")
-
-    fairness_grid_all = compute_jains_fairness(house_level_df["import_reduction"].values)
-    fairness_cost_all = compute_jains_fairness(house_level_df["cost_savings"].values)
-     
-    daily_summary_df = pd.DataFrame(daily_summaries)
-
-    total_cost_savings_all = daily_summary_df["cost_savings_abs"].sum()
-    total_baseline_cost_all = all_days_df.groupby('day')['baseline_cost'].sum().sum()
-    pct_cost_savings_all = total_cost_savings_all / total_baseline_cost_all if total_baseline_cost_all > 0 else 0.0
-
-    total_grid_reduction_all = daily_summary_df["grid_reduction_abs"].sum()
-    total_baseline_import_all = all_days_df.groupby('day')['grid_import_no_p2p'].sum().sum()
-    pct_grid_reduction_all = total_grid_reduction_all / total_baseline_import_all if total_baseline_import_all > 0 else 0.0
-
-    total_degradation_cost_all = daily_summary_df["total_degradation_cost"].sum()
-
-    # Calculate alternative performance metrics
-
-    # Grid Reduction During Solar Hours
-    agg_solar_per_step = all_days_df.groupby(['day', 'step'])['total_solar'].sum()
-    sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents)
-    sunny_df = all_days_df.set_index(['day', 'step'])[sunny_steps_mask].reset_index()
-    baseline_import_sunny = sunny_df['grid_import_no_p2p'].sum()
-    actual_import_sunny = sunny_df['grid_import_with_p2p'].sum()
-    grid_reduction_sunny_pct = 0.0
-    if baseline_import_sunny > 0:
-        grid_reduction_sunny_pct = (baseline_import_sunny - actual_import_sunny) / baseline_import_sunny
-
-    # Community Sourcing Rate
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    total_procured_energy = total_p2p_buy + total_actual_grid_import
-    community_sourcing_rate_pct = 0.0
-    if total_procured_energy > 0:
-        community_sourcing_rate_pct = total_p2p_buy / total_procured_energy
-
-    # Solar Sharing Efficiency
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    total_excess_solar = total_p2p_sell + total_grid_export
-    solar_sharing_efficiency_pct = 0.0
-    if total_excess_solar > 0:
-        solar_sharing_efficiency_pct = total_p2p_sell / total_excess_solar
-
-    # Cost savings in sunny hours
-    baseline_cost_sunny = sunny_df['baseline_cost'].sum()
-    actual_cost_sunny = sunny_df['actual_cost'].sum()
-    cost_savings_sunny_pct = (baseline_cost_sunny - actual_cost_sunny) / baseline_cost_sunny if baseline_cost_sunny > 0 else 0.0
-
-    total_p2p_buy = all_days_df['p2p_buy'].sum()
-    total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-    community_sourcing_rate_pct = total_p2p_buy / (total_p2p_buy + total_actual_grid_import) if (total_p2p_buy + total_actual_grid_import) > 0 else 0.0
-
-    total_p2p_sell = all_days_df['p2p_sell'].sum()
-    total_grid_export = all_days_df['grid_export'].sum()
-    solar_sharing_efficiency_pct = total_p2p_sell / (total_p2p_sell + total_grid_export) if (total_p2p_sell + total_grid_export) > 0 else 0.0
-
-    final_row = {
-        "day": "ALL_DAYS_SUMMARY",
-        "cost_savings_abs": total_cost_savings_all,
-        "cost_savings_pct": pct_cost_savings_all,
-        "grid_reduction_abs": total_grid_reduction_all,
-        "grid_reduction_pct": pct_grid_reduction_all,
-        "fairness_cost_savings": fairness_cost_all,
-        "fairness_grid_reduction": fairness_grid_all,
-        "total_degradation_cost": total_degradation_cost_all,
-        "grid_reduction_sunny_hours_pct": grid_reduction_sunny_pct,
-        "community_sourcing_rate_pct": community_sourcing_rate_pct,
-        "solar_sharing_efficiency_pct": solar_sharing_efficiency_pct,
-    }
-     
-    for col in daily_summary_df.columns:
-        if col not in final_row:
-            final_row[col] = np.nan
-    final_row_df = pd.DataFrame([final_row])
-
-    daily_summary_df = pd.concat([daily_summary_df, final_row_df], ignore_index=True)
-    summary_csv = os.path.join(logs_dir, "summary_per_day.csv")
-    daily_summary_df.to_csv(summary_csv, index=False)
-    print(f"Saved day-level summary with final multi-day row to: {summary_csv}")
-
-    # Final summary printout
-    print("\n================== EVALUATION SUMMARY ==================")
-    print(f"Evaluation finished for {days_to_evaluate} days.\n")
-     
-    print("--- Standard Metrics (24-Hour Average) ---")
-    print(f"Total grid reduction: {total_grid_reduction_all:.2f} kWh ({pct_grid_reduction_all:.2%})")
-    print(f"Total cost savings: ${total_cost_savings_all:.2f} ({pct_cost_savings_all:.2%})")
-    print(f"Jain's fairness on grid reduction: {fairness_grid_all:.3f}")
-    print(f"Jain's fairness on cost savings:   {fairness_cost_all:.3f}\n")
-
-    print("--- Alternative Metrics (Highlighting Peak Performance) ---")
-    print(f"Grid reduction during solar hours: {grid_reduction_sunny_pct:.2%}")
-    print(f"Cost savings during solar hours:   {cost_savings_sunny_pct:.2%}") 
-    print(f"Community sourcing rate:           {community_sourcing_rate_pct:.2%}")
-    print(f"Solar sharing efficiency:          {solar_sharing_efficiency_pct:.2%}")
-     
-    print("=========================================================")
-
-    # Plots
-    plot_daily_df = daily_summary_df[daily_summary_df["day"] != "ALL_DAYS_SUMMARY"].copy()
-    plot_daily_df["day"] = plot_daily_df["day"].astype(int)
-
-    # Daily Cost Savings Percentage
-    plt.figure(figsize=(12, 6))
-    plt.bar(plot_daily_df["day"], plot_daily_df["cost_savings_pct"] * 100, color='skyblue')
-    plt.xlabel("Day")
-    plt.ylabel("Cost Savings (%)")
-    plt.title("Daily Community Cost Savings Percentage")
-    plt.xticks(plot_daily_df["day"])
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_cost_savings_percentage.png"))
-    plt.close()
-
-    # Daily Total Demand vs. Solar
-    plt.figure(figsize=(12, 6))
-    bar_width = 0.4
-    days = plot_daily_df["day"]
-    plt.bar(days - bar_width/2, plot_daily_df["day_total_demand"], width=bar_width, label="Total Demand", color='coral')
-    plt.bar(days + bar_width/2, plot_daily_df["day_total_solar"], width=bar_width, label="Total Solar Generation", color='gold')
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Total Community Demand vs. Solar Generation Per Day")
-    plt.xticks(days)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_demand_vs_solar.png"))
-    plt.close()
-
-    # Combined Time Series of Energy Flows
-    step_group = all_days_df.groupby(["day", "step"]).sum(numeric_only=True).reset_index()
-    step_group["global_step"] = (step_group["day"] - 1) * env.num_steps + step_group["step"]
-     
-    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), sharex=True)
-     
-    # Subplot 1: Grid Import vs P2P Buy
-    ax1.plot(step_group["global_step"], step_group["grid_import_with_p2p"], label="Grid Import (with P2P)", color='r')
-    ax1.plot(step_group["global_step"], step_group["p2p_buy"], label="P2P Buy", color='g')
-    ax1.set_ylabel("Energy (kWh)")
-    ax1.set_title("Community Energy Consumption: Grid Import vs. P2P Buy")
-    ax1.legend()
-    ax1.grid(True, linestyle='--', alpha=0.6)
-
-    # Subplot 2: Grid Export vs P2P Sell
-    ax2.plot(step_group["global_step"], step_group["grid_export"], label="Grid Export", color='orange')
-    ax2.plot(step_group["global_step"], step_group["p2p_sell"], label="P2P Sell", color='b')
-    ax2.set_xlabel("Global Timestep")
-    ax2.set_ylabel("Energy (kWh)")
-    ax2.set_title("Community Energy Generation: Grid Export vs. P2P Sell")
-    ax2.legend()
-    ax2.grid(True, linestyle='--', alpha=0.6)
-     
-    plt.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "combined_energy_flows_timeseries.png"))
-    plt.close()
-
-    # Stacked Bar of Daily Energy Sources
-    daily_agg = all_days_df.groupby("day").sum(numeric_only=True)
-     
-    plt.figure(figsize=(12, 7))
-    plt.bar(daily_agg.index, daily_agg["grid_import_with_p2p"], label="Grid Import (with P2P)", color='crimson')
-    plt.bar(daily_agg.index, daily_agg["p2p_buy"], bottom=daily_agg["grid_import_with_p2p"], label="P2P Buy", color='limegreen')
-    plt.plot(daily_agg.index, daily_agg["grid_import_no_p2p"], label="Baseline Grid Import (No P2P)", color='blue', linestyle='--', marker='o')
-     
-    plt.xlabel("Day")
-    plt.ylabel("Energy (kWh)")
-    plt.title("Daily Energy Procurement: Baseline vs. P2P+Grid")
-    plt.xticks(daily_agg.index)
-    plt.legend()
-    plt.grid(axis='y', linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_energy_procurement_stacked.png"))
-    plt.close()
-
-    # Fairness Metrics Over Time
-    plt.figure(figsize=(12, 6))
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_cost_savings"], label="Cost Savings Fairness", marker='o')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_grid_reduction"], label="Grid Reduction Fairness", marker='s')
-    plt.plot(plot_daily_df["day"], plot_daily_df["fairness_reward"], label="Reward Fairness", marker='^')
-    plt.xlabel("Day")
-    plt.ylabel("Jain's Fairness Index")
-    plt.title("Daily Fairness Metrics")
-    plt.xticks(plot_daily_df["day"])
-    plt.ylim(0, 1.05)
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.7)
-    plt.savefig(os.path.join(plots_dir, "daily_fairness_metrics.png"))
-    plt.close()
-
-    # Per-House Savings and Reductions
-    fig, ax1 = plt.subplots(figsize=(15, 7))
-     
-    house_ids_str = house_level_df.index.astype(str)
-    bar_width = 0.4
-    index = np.arange(len(house_ids_str))
-
-    # Bar chart for cost savings
-    color1 = 'tab:green'
-    ax1.set_xlabel('House ID')
-    ax1.set_ylabel('Total Cost Savings ($)', color=color1)
-    ax1.bar(index - bar_width/2, house_level_df["cost_savings"], bar_width, label='Cost Savings', color=color1)
-    ax1.tick_params(axis='y', labelcolor=color1)
-    ax1.set_xticks(index)
-    ax1.set_xticklabels(house_ids_str, rotation=45, ha="right")
-     
-    # Second y-axis for grid import reduction
-    ax2 = ax1.twinx()
-    color2 = 'tab:blue'
-    ax2.set_ylabel('Total Grid Import Reduction (kWh)', color=color2)
-    ax2.bar(index + bar_width/2, house_level_df["import_reduction"], bar_width, label='Import Reduction', color=color2)
-    ax2.tick_params(axis='y', labelcolor=color2)
-
-    plt.title(f'Total Cost Savings & Grid Import Reduction Per House (over {days_to_evaluate} days)')
-    fig.tight_layout()
-    plt.savefig(os.path.join(plots_dir, "per_house_summary.png"))
-    plt.close()
-     
-    # Price Dynamics for a Single Day
-    day1_prices = all_days_df[all_days_df['day'] == 1][['step', 'grid_price', 'peer_price']].drop_duplicates()
-    plt.figure(figsize=(12, 6))
-    plt.plot(day1_prices['step'], day1_prices['grid_price'], label='Grid Price', color='darkorange')
-    plt.plot(day1_prices['step'], day1_prices['peer_price'], label='P2P Price', color='teal')
-    plt.xlabel("Timestep of Day")
-    plt.ylabel("Price ($/kWh)")
-    plt.title("Price Dynamics on Day 1")
-    plt.legend()
-    plt.grid(True, linestyle='--', alpha=0.6)
-    plt.savefig(os.path.join(plots_dir, "price_dynamics_day1.png"))
-    plt.close()
-     
-    # Battery State of Charge for Sample Houses
-    day1_df = all_days_df[all_days_df['day'] == 1]
-    battery_houses = day1_df.dropna(subset=['soc'])['house'].unique()
-     
-    if len(battery_houses) > 0:
-        sample_houses = battery_houses[:min(4, len(battery_houses))]
-        plt.figure(figsize=(12, 6))
-        for house in sample_houses:
-            house_df = day1_df[day1_df['house'] == house]
-            plt.plot(house_df['step'], house_df['soc'] * 100, label=f'House {house}')
-         
-        plt.xlabel("Timestep of Day")
-        plt.ylabel("State of Charge (%)")
-        plt.title("Battery SoC on Day 1 for Sample Houses")
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.6)
-        plt.savefig(os.path.join(plots_dir, "soc_dynamics_day1.png"))
-        plt.close()
-
-    print("All plots have been generated and saved. Evaluation complete.")
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/meanfield/meanfield_train.py

@@ -1,386 +0,0 @@
-import os
-import sys
-import re
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-import pandas as pd
-import time
-from datetime import datetime
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from solar_sys_environment import SolarSys
-from meanfield.trainer.mfac import MeanField
- 
-def main():
-
-    STATE_TO_RUN = "pennsylvania"  # "pennsylvania" or "colorado" or "oklahoma"
-
-    # Set the path to your training data
-    DATA_FILE_PATH = "/path/to/project/training/100houses_152days_TRAIN.csv"
-    num_episodes = 10000
-    batch_size = 256
-    checkpoint_interval = 100000
-    window_size = 32
-
-    env = SolarSys(
-        data_path=DATA_FILE_PATH,
-        state=STATE_TO_RUN,
-        time_freq="3H"
-    )
-
-    # Sanity check: env I/O shapes
-    print("Observation space:", env.observation_space)
-    print("Action space     :", env.action_space)
-
-    # Reset and inspect obs
-    obs = env.reset()
-    print(f"Reset returned {len(obs)} agent observations; each obs shape: {np.array(obs).shape}")
-
-    # Sample random actions and do one step
-    dummy_actions = np.random.rand(env.num_agents, env.action_space.shape[1]).astype(np.float32)
-    next_obs, rewards, done, info = env.step(dummy_actions)
-    print(f"Step outputs → next_obs: {len(next_obs)}×{np.array(next_obs).shape[1]}, "
-          f"rewards: {len(rewards)}, done: {done}")
-    print("Info keys:", list(info.keys()))
-
-    # Count the number of houses in each group
-    env.group_counts = {
-        0: env.agent_groups.count(0),
-        1: env.agent_groups.count(1)
-    }
-    print(f"Number of houses in each group: {env.group_counts}")
-
-    max_steps = env.num_steps
-
-    # Dims from the env
-    num_agents = env.num_agents
-    local_state_dim = env.observation_space.shape[1]
-    action_dim = env.action_space.shape[1]
-
-    # Build a unique run directory
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"meanfield_{STATE_TO_RUN}_{num_agents}agents_{num_episodes}eps_{timestamp}"
-    root_dir = os.path.join("Training_for_granularity", run_name)
-    os.makedirs(root_dir, exist_ok=True)
-    print(f"Saving training outputs to: {root_dir}")
-
-    logs_dir = os.path.join(root_dir, "logs")
-    plots_dir = os.path.join(root_dir, "plots")
-    os.makedirs(logs_dir, exist_ok=True)
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Create the MeanField agent
-    meanfield = MeanField(
-        n_agents=num_agents,
-        local_dim=local_state_dim,
-        global_dim=num_agents * local_state_dim,
-        act_dim=action_dim,
-        lr=2e-4,
-        gamma=0.95,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=4,
-        batch_size=batch_size
-    )
-
-    # Tracking / Logging Variables
-    episode_rewards = []
-    episode_total_rewards = []
-    block_mean_rewards = []
-    block_total_rewards = []
-
-    agent_rewards_log = [[] for _ in range(num_agents)]
-    best_mean_reward = -1e9
-    best_model_path = os.path.join(logs_dir, "best_model.pth")
-
-    daily_rewards = []
-    monthly_rewards = []
-
-    training_start_time = time.time()
-    episode_durations = []
-    total_steps_global = 0
-    episode_log_data = []
-    performance_metrics_log = []
-
-    agent_charge_log = [[] for _ in range(num_agents)]
-    agent_discharge_log = [[] for _ in range(num_agents)]
-
-    # Training Loop
-    for episode in range(1, num_episodes + 1):
-        episode_start_time = time.time()
-
-        obs = np.array(env.reset(), dtype=np.float32)
-
-        # Collect metrics from the previous episode
-        if episode > 1:
-            last_episode_metrics = env.get_episode_metrics()
-            last_episode_metrics['Episode'] = episode - 1
-            performance_metrics_log.append(last_episode_metrics)
-
-        total_reward = np.zeros(num_agents, dtype=np.float32)
-        done = False
-        step_count = 0
-        day_logs = []
-        episode_charges = [[] for _ in range(num_agents)]
-        episode_discharges = [[] for _ in range(num_agents)]
-
-        while not done:
-            # Build global state and pick actions
-            global_obs = obs.flatten()
-            actions, logps = meanfield.select_action(obs, global_obs)
-
-            # Step environment
-            next_obs_list, rewards, done, info = env.step(actions)
-
-            # Convert next observations to NumPy array
-            next_obs = np.array(next_obs_list, dtype=np.float32)
-            next_global_obs = next_obs.flatten()
-
-            # Store transition
-            local_obs_arr = np.array(obs, dtype=np.float32)
-
-            meanfield.store(
-                local_obs_arr,
-                global_obs,
-                actions,
-                logps,
-                rewards,
-                done,
-                next_global_obs
-            )
-            total_reward += rewards
-            obs = next_obs
-            step_count += 1
-            total_steps_global += 1
-
-            day_logs.append({
-                "step": step_count - 1,
-                "grid_import_no_p2p": info["grid_import_no_p2p"],
-                "grid_import_with_p2p": info["grid_import_with_p2p"],
-                "p2p_buy": info["p2p_buy"],
-                "p2p_sell": info["p2p_sell"],
-                "costs": info["costs"],
-                "charge_amount": info.get("charge_amount", np.zeros(num_agents)),
-                "discharge_amount": info.get("discharge_amount", np.zeros(num_agents))
-            })
-
-            if step_count >= max_steps:
-                break
-
-        # After each episode
-        # Compute per-episode metrics
-        sum_ep_reward = float(np.sum(total_reward))
-        mean_ep_reward = float(np.mean(total_reward))
-
-        episode_total_rewards.append(sum_ep_reward)
-        episode_rewards.append(mean_ep_reward)
-        daily_rewards.append(mean_ep_reward)
-
-        # If we just finished a block of window_size episodes, aggregate
-        if len(daily_rewards) % window_size == 0:
-            last_totals = episode_total_rewards[-window_size:]
-            block_sum = sum(last_totals)
-            block_total_rewards.append(block_sum)
-
-            last_means = daily_rewards[-window_size:]
-            block_mean = sum(last_means) / window_size
-            block_mean_rewards.append(block_mean)
-
-            block_idx = len(block_mean_rewards)
-            print(
-                f"→ Completed Block {block_idx} "
-                f"| Episodes {(block_idx-1)*window_size + 1}–{block_idx*window_size} "
-                f"| Block Total Reward: {block_sum:.3f} "
-                f"| Block Mean Reward: {block_mean:.3f}"
-            )
-
-        # Log agent-level rewards
-        for i in range(num_agents):
-            agent_rewards_log[i].append(total_reward[i])
-            episode_charges[i].append(actions[i][4])
-            episode_discharges[i].append(actions[i][5])
-
-        # Summarize P2P steps
-        steps_data = []
-        for entry in day_logs:
-            step_idx = entry["step"]
-            p2p_buy_array = entry["p2p_buy"]
-            p2p_sell_array = entry["p2p_sell"]
-            grid_no_p2p_array = entry["grid_import_no_p2p"]
-            grid_with_p2p_array = entry["grid_import_with_p2p"]
-
-            steps_data.append({
-                "step": step_idx,
-                "p2p_buy_sum": float(np.sum(p2p_buy_array)),
-                "p2p_sell_sum": float(np.sum(p2p_sell_array)),
-                "grid_import_no_p2p_sum": float(np.sum(grid_no_p2p_array)),
-                "grid_import_with_p2p_sum": float(np.sum(grid_with_p2p_array))
-            })
-
-        baseline_cost = np.sum([np.sum(entry["grid_import_no_p2p"]) * env.get_grid_price(entry["step"])
-                                for entry in day_logs])
-        actual_cost = np.sum([np.sum(entry["costs"]) for entry in day_logs])
-        cost_reduction = (baseline_cost - actual_cost) / baseline_cost
-
-        # Update the meanfield agent
-        meanfield.update()
-
-        # Save if best
-        if mean_ep_reward > best_mean_reward:
-            best_mean_reward = mean_ep_reward
-            meanfield.save(best_model_path)
-
-        if episode % checkpoint_interval == 0:
-            ckpt_path = os.path.join(logs_dir, f"checkpoint_{episode}.pth")
-            meanfield.save(ckpt_path)
-
-        episode_end_time = time.time()
-        episode_duration = episode_end_time - episode_start_time
-
-        print(
-            f"Episode {episode}/{num_episodes} "
-            f"| Time per Episode: {episode_duration:.2f}s "
-            f"| Steps: {step_count} "
-            f"| Mean Reward: {mean_ep_reward:.3f} "
-            f"| Cost Reduction: {cost_reduction:.2%}"
-        )
-
-        # Record data in per-episode log
-        episode_log_data.append({
-            "Episode": episode,
-            "Steps": step_count,
-            "Mean_Reward": mean_ep_reward,
-            "Total_Reward": sum_ep_reward,
-            "Cost_Reduction_Pct": cost_reduction * 100,
-            "Baseline_Cost": baseline_cost,
-            "Actual_Cost": actual_cost,
-            "Episode_Duration": episode_duration,
-            "Total_Charge": np.sum([np.sum(entry["charge_amount"]) for entry in day_logs]),
-            "Total_Discharge": np.sum([np.sum(entry["discharge_amount"]) for entry in day_logs])
-        })
-
-        for i in range(num_agents):
-            agent_charge_log[i].append(np.mean(episode_charges[i]))
-            agent_discharge_log[i].append(np.mean(episode_discharges[i]))
-
-    # Capture the final episode's metrics
-    final_episode_metrics = env.get_episode_metrics()
-    final_episode_metrics['Episode'] = num_episodes
-    performance_metrics_log.append(final_episode_metrics)
-
-    # End of all training
-    training_end_time = time.time()
-    total_training_time = training_end_time - training_start_time
-
-    # Save out per-episode agent rewards + mean rewards
-    np.save(os.path.join(logs_dir, "agent_rewards.npy"), np.array(agent_rewards_log))
-    np.save(os.path.join(logs_dir, "mean_rewards.npy"), np.array(episode_rewards))
-    np.save(os.path.join(logs_dir, "total_rewards.npy"), np.array(episode_total_rewards))
-
-    # Create Final DataFrame for Logging and Plotting
-    df_rewards_log = pd.DataFrame(episode_log_data)
-    df_perf_log = pd.DataFrame(performance_metrics_log)
-
-    # Merge the two DataFrames on the 'Episode' column
-    df_final_log = pd.merge(df_rewards_log, df_perf_log.drop(columns=[
-        'degradation_cost_over_time',
-        'cost_savings_over_time',
-        'grid_reduction_over_time'
-    ]), on="Episode")
-
-    # PLOTTING
-    os.makedirs(plots_dir, exist_ok=True)
-
-    # Helper: centered moving average
-    def moving_avg(series, window):
-        return pd.Series(series).rolling(window=window, center=True, min_periods=1).mean().to_numpy()
-
-    # Smoothing window (in episodes)
-    ma_window = 300
-    episodes = np.arange(1, num_episodes + 1)
-
-    # Mean Reward moving average
-    reward_ma = moving_avg(df_final_log["Mean_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, reward_ma, linewidth=2, label=f"Mean Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Mean Reward")
-    plt.title("meanfield: Mean Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "mean_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Total Reward moving average
-    total_ma = moving_avg(df_final_log["Total_Reward"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, total_ma, linewidth=2, label=f"Total Reward MA (win={ma_window})")
-    plt.xlabel("Episode")
-    plt.ylabel("Total Reward")
-    plt.title("meanfield: Total Reward Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "total_reward_ma.png"), dpi=200)
-    plt.close()
-
-    # Cost Reduction (%) moving average
-    cost_ma = moving_avg(df_final_log["Cost_Reduction_Pct"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, cost_ma, linewidth=2, label="Cost Reduction MA (%)")
-    plt.xlabel("Episode")
-    plt.ylabel("Cost Reduction (%)")
-    plt.title("meanfield: Cost Reduction Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "cost_reduction_ma.png"), dpi=200)
-    plt.close()
-
-    # Battery Degradation Cost moving average
-    degradation_ma = moving_avg(df_final_log["battery_degradation_cost_total"], ma_window)
-    plt.figure(figsize=(8, 5))
-    plt.plot(episodes, degradation_ma, linewidth=2, label=f"Degradation Cost MA (win={ma_window})", color='purple')
-    plt.xlabel("Episode")
-    plt.ylabel("Total Degradation Cost ($)")
-    plt.title("meanfield: Battery Degradation Cost Moving Average")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "degradation_cost_ma.png"), dpi=200)
-    plt.close()
-
-    print(f"\nAll moving-average plots saved to: {plots_dir}")
-
-    # Save Final Logs to CSV
-    total_time_row = pd.DataFrame([{
-        "Episode": "Total_Training_Time",
-        "Episode_Duration": total_training_time
-    }])
-    df_to_save = pd.concat([df_final_log, total_time_row], ignore_index=True)
-
-    log_csv_path = os.path.join(logs_dir, "training_performance_log.csv")
-
-    # Select and reorder columns for the final CSV
-    columns_to_save = [
-        "Episode",
-        "Mean_Reward",
-        "Total_Reward",
-        "Cost_Reduction_Pct",
-        "Episode_Duration",
-        "battery_degradation_cost_total",
-    ]
-    df_to_save = df_to_save[columns_to_save]
-
-    df_to_save.to_csv(log_csv_path, index=False)
-
-    print(f"Saved comprehensive training performance log to: {log_csv_path}")
-
-    # Final Timings Printout
-    print("\n" + "="*50)
-    print("TRAINING COMPLETE".center(50))
-    print(f"Total training time: {total_training_time:.2f} seconds")
-    print("="*50)
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/Flat_System/meanfield/trainer/mfac.py

@@ -1,219 +0,0 @@
-# meanfield.py 
-import torch
-import torch.nn as nn
-import random
-import numpy as np
-from torch.distributions import Normal
-from torch.amp import autocast
-from torch.cuda.amp import GradScaler
-
-#device selection
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-    print("Using CUDA (NVIDIA GPU)")
-else:
-    device = torch.device("cpu")
-    print("Using CPU")
-
-def set_global_seed(seed: int):
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)
-        torch.backends.cudnn.deterministic = False
-        torch.backends.cudnn.benchmark = True
-
-SEED = 42
-set_global_seed(SEED)
-
-class MLP(nn.Module):
-    def __init__(self, input_dim, hidden_dims, output_dim):
-        super().__init__()
-        layers = []
-        last_dim = input_dim
-        for h in hidden_dims:
-            layers += [nn.Linear(last_dim, h), nn.ReLU()]
-            last_dim = h
-        layers.append(nn.Linear(last_dim, output_dim))
-        self.net = nn.Sequential(*layers)
-
-    def forward(self, x):
-        return self.net(x)
-
-class Actor(nn.Module):
-    def __init__(self, obs_dim, act_dim, hidden=(64,64)):
-        super().__init__()
-        self.net = MLP(obs_dim, hidden, act_dim)
-        self.log_std = nn.Parameter(torch.zeros(act_dim))
-
-    def forward(self, x):
-        mean = self.net(x)
-        std = torch.exp(self.log_std)
-        return mean, std 
-
-class Critic(nn.Module):
-    def __init__(self, state_dim, hidden=(128,128)):
-        super().__init__()
-        self.net = MLP(state_dim, hidden, 1)
-
-    def forward(self, x):
-        return self.net(x).squeeze(-1)
-
-class MeanField:
-    def __init__(
-        self,
-        n_agents,
-        local_dim,
-        global_dim,
-        act_dim,
-        lr=3e-4,
-        gamma=0.99,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=10,
-        batch_size=1024,
-        episode_len=96
-    ):
-        self.n_agents = n_agents
-        self.local_dim = local_dim
-        self.global_dim = global_dim
-        self.act_dim = act_dim
-        self.gamma    = gamma
-        self.lam      = lam
-        self.clip_eps = clip_eps
-        self.k_epochs = k_epochs
-        self.batch_size = batch_size
-        self.episode_len = episode_len
-
-        self.actor  = Actor(local_dim + global_dim, act_dim).to(device)
-        self.critic = Critic(global_dim).to(device)
-
-        self.opt_a = torch.optim.Adam(self.actor.parameters(), lr=lr)
-        self.opt_c = torch.optim.Adam(self.critic.parameters(), lr=lr)
-
-        print("MeanField CUDA AMP is disabled for stability.")
-        
-        self.init_buffer()
-
-    def init_buffer(self):
-        self.ls_buf = np.zeros((self.episode_len, self.n_agents, self.local_dim), dtype=np.float32)
-        self.gs_buf = np.zeros((self.episode_len, self.global_dim), dtype=np.float32)
-        self.ac_buf = np.zeros((self.episode_len, self.n_agents, self.act_dim), dtype=np.float32)
-        self.lp_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float32)
-        self.rw_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float32)
-        self.done_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float32)
-        self.next_gs_buf = np.zeros((self.episode_len, self.global_dim), dtype=np.float32)
-        self.step_idx = 0
-
-    @torch.no_grad()
-    def select_action(self, local_obs, global_obs):
-        l = torch.from_numpy(local_obs).float().to(device)
-        g = torch.from_numpy(global_obs).float().to(device).unsqueeze(0).expand(self.n_agents, -1)
-        input_x = torch.cat([l, g], dim=-1)
-        mean, std = self.actor(input_x)
-        dist = Normal(mean, std)
-        a = dist.sample()
-        return a.cpu().numpy(), dist.log_prob(a).sum(-1).cpu().numpy()
-
-    def store(self, local_obs, global_obs, action, logp, reward, done, next_global_obs):
-        if self.step_idx < self.episode_len:
-            self.ls_buf[self.step_idx] = local_obs
-            self.gs_buf[self.step_idx] = global_obs
-            self.ac_buf[self.step_idx] = action
-            self.lp_buf[self.step_idx] = logp
-            self.rw_buf[self.step_idx] = reward
-            self.done_buf[self.step_idx] = done
-            self.next_gs_buf[self.step_idx] = next_global_obs
-            self.step_idx += 1
-
-    def compute_gae(self, T, vals):
-        """
-        Computes Generalized Advantage Estimation (GAE).
-        """
-        N = self.n_agents
-        adv_buf = np.zeros_like(self.rw_buf[:T])
-
-
-        if not self.done_buf[T-1].all():
-            with torch.no_grad():
-                v_last = self.critic(
-                    torch.from_numpy(self.next_gs_buf[T-1]).float().to(device)
-                ).cpu().numpy()
-        else:
-            v_last = 0.0
-        vals_agent = vals.unsqueeze(1).expand(-1, N).cpu().numpy()
-        rewards = self.rw_buf[:T]
-        masks = 1.0 - self.done_buf[:T]
-        gae = 0
-        for t in reversed(range(T)):
-            v_next = vals_agent[t+1] if t < T - 1 else v_last
-            delta = rewards[t] + self.gamma * v_next * masks[t] - vals_agent[t]
-            adv_buf[t] = gae = delta + self.gamma * self.lam * masks[t] * gae
-        ret_buf = adv_buf + vals_agent
-        adv_flat = torch.from_numpy(adv_buf.flatten()).float().to(device)
-        ret_flat = torch.from_numpy(ret_buf.flatten()).float().to(device)
-        return adv_flat, ret_flat
-
-    def update(self):
-        T = self.step_idx
-        if T == 0: return
-
-        gs_tensor = torch.from_numpy(self.gs_buf[:T]).float().to(device)
-        ls_tensor = torch.from_numpy(self.ls_buf[:T]).float().to(device).view(T * self.n_agents, -1)
-        ac_tensor = torch.from_numpy(self.ac_buf[:T]).float().to(device).view(T * self.n_agents, -1)
-        lp_tensor = torch.from_numpy(self.lp_buf[:T]).float().to(device).view(-1)
-        
-        with torch.no_grad():
-            vals = self.critic(gs_tensor)
-
-        adv_flat, ret_flat = self.compute_gae(T, vals)
-        adv_flat = (adv_flat - adv_flat.mean()) / (adv_flat.std() + 1e-8)
-
-        gs_for_batch = gs_tensor.unsqueeze(1).expand(-1, self.n_agents, -1).reshape(T * self.n_agents, self.global_dim)
-
-        dataset = torch.utils.data.TensorDataset(ls_tensor, gs_for_batch, ac_tensor, lp_tensor, adv_flat, ret_flat)
-        gen = torch.Generator()
-        gen.manual_seed(SEED)
-        loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, generator=gen)
-
-        for _ in range(self.k_epochs):
-            for b_ls, b_gs, b_ac, b_lp, b_adv, b_ret in loader:
-                input_a = torch.cat([b_ls, b_gs], dim=-1)
-                mean, std = self.actor(input_a)
-                dist = Normal(mean, std)
-
-                entropy = dist.entropy().mean()
-
-                lp_new = dist.log_prob(b_ac).sum(-1)
-                ratio = torch.exp(lp_new - b_lp)
-                surr1 = ratio * b_adv
-                surr2 = torch.clamp(ratio, 1 - self.clip_eps, 1 + self.clip_eps) * b_adv
-
-                actor_loss = -torch.min(surr1, surr2).mean() - 0.01 * entropy
-
-                self.opt_a.zero_grad()
-                actor_loss.backward()
-                nn.utils.clip_grad_norm_(self.actor.parameters(), max_norm=0.5)
-                self.opt_a.step()
-
-
-                val_pred = self.critic(b_gs)
-                critic_loss = nn.MSELoss()(val_pred, b_ret)
-
-                self.opt_c.zero_grad()
-                critic_loss.backward()
-                nn.utils.clip_grad_norm_(self.critic.parameters(), max_norm=0.5)
-                self.opt_c.step()
-
-        self.step_idx = 0
-        
-    def save(self, path):
-        torch.save({'actor': self.actor.state_dict(),
-                    'critic': self.critic.state_dict()}, path)
-
-    def load(self, path):
-        data = torch.load(path, map_location=device)
-        self.actor.load_state_dict(data['actor'])
-        self.critic.load_state_dict(data['critic'])

Other_algorithms/Flat_System/solar_sys_environment.py

@@ -1,635 +0,0 @@
-import gym
-import pandas as pd
-import numpy as np
-from collections import deque
-import random
-random.seed(42)
-np.random.seed(42)
-
-class SolarSys(gym.Env):
-
-    def __init__(
-        self,
-        data_path="/path/to/project/training/200houses_152days_TRAIN.csv",
-        state="oklahoma",  # for Oklahoma (example)
-        time_freq="15T",  # "15T", "30T", "1H", "3H", "6H"
-    ):
-        
-        super().__init__()
-        # Store config
-        self.data_path = data_path
-        self.time_freq = time_freq
-        self.state = state.lower()
-
-        # Centralized Pricing Configuration
-        self._pricing_info = {
-            "oklahoma": {
-                "max_grid_price": 0.2112,
-                "feed_in_tariff": 0.04,
-                "price_function": self._get_oklahoma_price
-            },
-            "colorado": {
-                "max_grid_price": 0.32,
-                "feed_in_tariff": 0.055,
-                "price_function": self._get_colorado_price
-            },
-            "pennsylvania": {
-                "max_grid_price": 0.12505,
-                "feed_in_tariff": 0.06,
-                "price_function": self._get_pennsylvania_price
-            }
-        }
-
-        if self.state not in self._pricing_info:
-            raise ValueError(f"State '{self.state}' is not supported. Available states: {list(self._pricing_info.keys())}")
-        
-        state_config = self._pricing_info[self.state]
-        self.max_grid_price = state_config["max_grid_price"]
-        self.feed_in_tariff = state_config["feed_in_tariff"]
-        self._get_price_function = state_config["price_function"]
-
-        try:
-            all_data = pd.read_csv(data_path)
-            all_data["local_15min"] = pd.to_datetime(all_data["local_15min"], utc=True)
-            all_data.set_index("local_15min", inplace=True)
-            all_data = all_data.resample(time_freq).mean() 
-
-        except FileNotFoundError:
-            raise FileNotFoundError(f"Data file {data_path} not found.")
-        except pd.errors.EmptyDataError:
-            raise ValueError(f"Data file {data_path} is empty.")
-        except Exception as e:
-            raise ValueError(f"Error loading data: {e}")
-
-        # Compute global maxima for normalization
-        grid_cols = [c for c in all_data.columns if c.startswith("grid_")]
-        solar_cols = [c for c in all_data.columns if c.startswith("total_solar_")]
-        all_grid = all_data[grid_cols].values
-        all_solar = all_data[solar_cols].values
-
-        # max total demand = max(grid + solar) over all time & agents
-        self.global_max_demand = float((all_grid + all_solar).max()) + 1e-8
-
-        # max solar generation alone
-        self.global_max_solar = float(all_solar.max()) + 1e-8
-
-        # Store the resampled dataset
-        self.all_data = all_data
-
-        self.time_freq = time_freq
-        freq_offset = pd.tseries.frequencies.to_offset(time_freq)
-        minutes_per_step = freq_offset.nanos / 1e9 / 60.0
-        self.steps_per_day = int(24 * 60 // minutes_per_step)
-
-        total_rows = len(self.all_data)
-        self.total_days = total_rows // self.steps_per_day
-        if self.total_days < 1:
-            raise ValueError(
-                f"After resampling, dataset has {total_rows} rows, which is "
-                f"less than a single day of {self.steps_per_day} steps."
-            )
-
-        self.house_ids = [
-            col.split("_")[1] for col in self.all_data.columns
-            if col.startswith("grid_")
-        ]
-        self.num_agents = len(self.house_ids)
-        self.original_no_p2p_import = {}
-        for hid in self.house_ids:
-            col_grid = f"grid_{hid}"
-            self.original_no_p2p_import[hid] = self.all_data[col_grid].clip(lower=0.0).values
-        
-        # Determine population groups
-        # group 1 = has any solar; group 0 = never solar
-        solar_cols = [f"total_solar_{hid}" for hid in self.house_ids]
-        solar_sums = self.all_data[solar_cols].sum(axis=0).to_dict()
-        self.agent_groups = [
-            1 if solar_sums[f"total_solar_{hid}"] > 0 else 0
-            for hid in self.house_ids
-        ]
-
-        # Count the number of houses in each group
-        self.group_counts = {
-            0: self.agent_groups.count(0),
-            1: self.agent_groups.count(1)
-        }
-        print(f"Number of houses in each group: {self.group_counts}")
-
-        # Battery logic
-        self.battery_options = {
-            "teslapowerwall": {"max_capacity": 13.5, "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 5.0, "max_discharge_rate": 5.0, "degradation_cost_per_kwh": 0.005},
-            "enphase":         {"max_capacity": 5.0,  "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 2.0, "max_discharge_rate": 2.0, "degradation_cost_per_kwh": 0.005},
-            "franklin":        {"max_capacity": 15.0, "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 6.0, "max_discharge_rate": 6.0, "degradation_cost_per_kwh": 0.005},
-        }
-        
-        # Identify which houses actually have solar
-        self.solar_houses = [
-            hid for hid in self.house_ids
-            if (self.all_data[f"total_solar_{hid}"] > 0).any()
-        ]
-
-        # Assign a random battery type to each solar-equipped house
-        self.batteries = {}
-        for hid in self.solar_houses:
-            choice = random.choice(list(self.battery_options))
-            specs = self.battery_options[choice]
-            self.batteries[hid] = {"soc": 0.0, **specs}
-
-        # Observation & Action Spaces
-        # [own_demand, own_solar, grid_price, peer_price,
-        #  total_demand_others, total_solar_others, SOC, time_of_day]
-        self.observation_space = gym.spaces.Box(
-            low=-np.inf, high=np.inf,
-            shape=(self.num_agents, 8),
-            dtype=np.float32
-        )
-        
-        # [sell_to_grid, buy_from_grid, sell_to_peers, buy_from_peers, charge_battery, discharge_battery]
-        self.action_space = gym.spaces.Box(
-            low=0.0,
-            high=1.0,
-            shape=(self.num_agents, 6),
-            dtype=np.float32
-        )
-        
-        self.episode_metrics = {}
-        self._initialize_episode_metrics()
-        
-        # Initialize episode variables
-        self.data = None
-        self.env_log = []
-        self.day_index = -1  
-        self.current_step = 0
-        self.num_steps = self.steps_per_day  
-        self.demands = {}
-        self.solars = {}
-        self.previous_actions = {
-            hid: np.zeros(6) for hid in self.house_ids
-        }
-
-
-    def _initialize_episode_metrics(self):
-        """Initialize or reset all metrics tracked over a single episode."""
-        self.cumulative_grid_reduction = 0.0
-        self.cumulative_grid_reduction_peak = 0.0
-        self.cumulative_degradation_cost = 0.0
-        self.agent_cost_savings = np.zeros(self.num_agents)
-        self.degradation_cost_timeseries = []
-        self.cost_savings_timeseries = []
-        self.grid_reduction_timeseries = []
-
-
-    # Price Functions
-    def get_grid_price(self, step_idx):
-        """Return grid price for the current step based on selected state."""
-        return self._get_price_function(step_idx)
-
-
-    def _get_oklahoma_price(self, step_idx):
-        # Oklahoma Gas & Electric (OG&E) TOU
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        # Peak: 2 pm to 7 pm
-        if 14 <= hour < 19:
-            return 0.2112
-        # Off-peak: All other times
-        else:
-            return 0.0434
-
-
-    def _get_colorado_price(self, step_idx):
-        # Xcel Energy Colorado TOU
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        # On-peak: 3 pm to 7 pm
-        if 15 <= hour < 19:
-            return 0.32
-        # Mid-peak: 1 pm to 3 pm
-        elif 13 <= hour < 15:
-            return 0.22
-        # Off-peak: Before 1 pm and after 7 pm
-        else:
-            return 0.12
-
-
-    def _get_pennsylvania_price(self, step_idx):
-        # Duquesne Light (Pennsylvania) EV TOU
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        # Peak: 1 pm to 9 pm
-        if 13 <= hour < 21:
-            return 0.125048
-        # Super Off-Peak: 11 pm to 6 am
-        elif hour >= 23 or hour < 6:
-            return 0.057014
-        # Off-Peak: 6 am to 1 pm and 9 pm to 11 pm
-        else:
-            return 0.079085
-
-
-    def get_peer_price(self, step_idx, total_surplus, total_shortfall):
-        grid_price = self.get_grid_price(step_idx)
-        feed_in_tariff = self.feed_in_tariff 
-
-        base_price = grid_price * 0.90
-        net_demand = total_shortfall - total_surplus
-        total_potential_trade = total_shortfall + total_surplus + 1e-6
-        elasticity_factor = 0.3
-        price_multiplier = np.exp(elasticity_factor * (net_demand / total_potential_trade))
-        peer_price = base_price * price_multiplier
-        final_price = float(np.clip(peer_price, feed_in_tariff, grid_price))
-
-        return final_price
-
-
-    def reset(self):
-        # Finalize and store metrics from completed episode before resetting
-        if self.current_step > 0:
-            positive_savings = self.agent_cost_savings[self.agent_cost_savings > 0]
-            if len(positive_savings) > 1:
-                fairness_on_savings = self._compute_jains_index(positive_savings)
-            else:
-                fairness_on_savings = 0.0
-
-            # Store all final metrics
-            self.episode_metrics = {
-                "grid_reduction_entire_day": self.cumulative_grid_reduction,
-                "grid_reduction_peak_hours": self.cumulative_grid_reduction_peak,
-                "total_cost_savings": np.sum(self.agent_cost_savings),
-                "fairness_on_cost_savings": fairness_on_savings,
-                "battery_degradation_cost_total": self.cumulative_degradation_cost,
-                "degradation_cost_over_time": self.degradation_cost_timeseries,
-                "cost_savings_over_time": self.cost_savings_timeseries,
-                "grid_reduction_over_time": self.grid_reduction_timeseries,
-            }
-        
-        self.day_index = np.random.randint(0, self.total_days)
-
-        start_row = self.day_index * self.steps_per_day
-        end_row = start_row + self.steps_per_day
-        day_data = self.all_data.iloc[start_row:end_row].copy()
-        self.data = day_data  
-
-        self.no_p2p_import_day = {}
-        for hid in self.house_ids:
-            self.no_p2p_import_day[hid] = self.original_no_p2p_import[hid][start_row:end_row]
-
-        self.demands = {}
-        self.solars = {}
-
-        for hid in self.house_ids:
-            col_grid = f"grid_{hid}"
-            col_solar = f"total_solar_{hid}"
-
-            grid_series = day_data[col_grid].fillna(0.0)
-            solar_series = day_data[col_solar].fillna(0.0).clip(lower=0.0)
-
-            demand_array = grid_series.values + solar_series.values
-            demand_array = np.clip(demand_array, 0.0, None)
-
-            self.demands[hid] = demand_array
-            self.solars[hid] = solar_series.values
-
-        self.current_step = 0
-        self.env_log = []
-        
-        # Reset previous_actions to 6 zeros
-        for hid in self.house_ids:
-            self.previous_actions[hid] = np.zeros(6)
-        
-        self._initialize_episode_metrics()
-
-        # Randomize battery SOC between 30%–70% of capacity
-        for hid, batt in self.batteries.items():
-            low = 0.30 * batt["max_capacity"]
-            high = 0.70 * batt["max_capacity"]
-            batt["soc"] = random.uniform(low, high)
-
-        obs = self._get_obs()
-        obs_list = [obs[i] for i in range(self.num_agents)]
-        return obs_list
-
-
-    def step(self, actions):
-        # Validate & clamp actions
-        actions = np.array(actions, dtype=np.float32)
-        if actions.shape != (self.num_agents, 6):
-            raise ValueError(f"Actions shape mismatch: got {actions.shape}, expected {(self.num_agents, 6)}")
-        actions = np.clip(actions, 0.0, 1.0)
-
-        a_sellGrid = actions[:, 0]
-        a_buyGrid = actions[:, 1]
-        a_sellPeers = actions[:, 2]
-        a_buyPeers = actions[:, 3]
-        a_chargeBatt = actions[:, 4]
-        a_dischargeBatt = actions[:, 5]
-        
-        # Gather current demand & solar
-        demands = []
-        solars = []
-        for i, hid in enumerate(self.house_ids):
-            demands.append(self.demands[hid][self.current_step])
-            solars.append(self.solars[hid][self.current_step])
-
-        demands = np.array(demands, dtype=np.float32)
-        solars = np.array(solars, dtype=np.float32)
-
-        # Calculations for peer_price and grid_price
-        total_surplus = np.maximum(solars - demands, 0.0).sum()
-        total_shortfall = np.maximum(demands - solars, 0.0).sum()
-        peer_price = self.get_peer_price(self.current_step, total_surplus, total_shortfall)
-        grid_price = self.get_grid_price(self.current_step)
-
-        # Enforce "self-use first"
-        shortfall = np.maximum(demands - solars, 0.0)
-        surplus = np.maximum(solars - demands, 0.0)
-        
-        final_shortfall = shortfall.copy()
-        final_surplus = surplus.copy()
-        grid_import = np.zeros(self.num_agents, dtype=np.float32)
-        grid_export = np.zeros(self.num_agents, dtype=np.float32)
-
-        # Battery discharge
-        discharge_amount = np.zeros(self.num_agents, dtype=np.float32)
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                batt = self.batteries[hid]
-                max_dis = batt["max_discharge_rate"]
-                available = batt["soc"] * batt["discharge_efficiency"]
-                desired = a_dischargeBatt[i] * max_dis
-                actual = min(desired, available, final_shortfall[i])
-                batt["soc"] -= actual / batt["discharge_efficiency"]
-                final_shortfall[i] -= actual
-                discharge_amount[i] = actual
-
-        # Battery charge
-        charge_amount = np.zeros(self.num_agents, dtype=np.float32)
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                batt = self.batteries[hid]
-                max_ch = batt["max_charge_rate"]
-                cap_left = batt["max_capacity"] - batt["soc"]
-                desired = a_chargeBatt[i] * max_ch
-                actual = min(desired, cap_left / batt["charge_efficiency"], final_surplus[i])
-                batt["soc"] += actual * batt["charge_efficiency"]
-                final_surplus[i] -= actual
-                charge_amount[i] = actual
-        
-        # P2P matching
-        battery_offer = np.zeros(self.num_agents, dtype=np.float32)
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                battery_offer[i] = self.batteries[hid]["soc"] * self.batteries[hid]["discharge_efficiency"]
-        effective_surplus = final_surplus + battery_offer
-        
-        netPeer = a_buyPeers - a_sellPeers
-        p2p_buy_request = np.zeros(self.num_agents, dtype=np.float32)
-        p2p_sell_offer = np.zeros(self.num_agents, dtype=np.float32)
-        for i in range(self.num_agents):
-            if netPeer[i] > 0:
-                p2p_buy_request[i] = netPeer[i] * final_shortfall[i]
-            elif netPeer[i] < 0:
-                p2p_sell_offer[i] = -netPeer[i] * effective_surplus[i]
-                
-        total_sell = np.sum(p2p_sell_offer)
-        total_buy = np.sum(p2p_buy_request)
-        matched = min(total_sell, total_buy)
-        
-        if matched > 1e-9:
-            sell_fraction = p2p_sell_offer / (total_sell + 1e-12)
-            buy_fraction = p2p_buy_request / (total_buy + 1e-12)
-            actual_sold = matched * sell_fraction
-            actual_bought = matched * buy_fraction
-        else:
-            actual_sold = np.zeros(self.num_agents, dtype=np.float32)
-            actual_bought = np.zeros(self.num_agents, dtype=np.float32)
-            
-        from_batt_p2p = np.minimum(actual_sold, battery_offer)
-        from_solar_p2p = actual_sold - from_batt_p2p
-
-        # Update balances
-        final_surplus -= from_solar_p2p
-        final_shortfall -= actual_bought
-
-        # Deduct peer battery sales from SOC
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                from_batt = min(actual_sold[i], battery_offer[i])
-                self.batteries[hid]["soc"] -= from_batt / self.batteries[hid]["discharge_efficiency"]
-                self.batteries[hid]["soc"] = max(0.0, self.batteries[hid]["soc"])
-                
-        # Grid trades
-        netGrid = a_buyGrid - a_sellGrid
-        for i in range(self.num_agents):
-            if netGrid[i] > 0:
-                grid_import[i] = netGrid[i] * final_shortfall[i]
-            elif netGrid[i] < 0:
-                grid_export[i] = -netGrid[i] * final_surplus[i]
-        forced = np.maximum(final_shortfall - grid_import, 0.0)
-        grid_import += forced
-            
-        # Calculate costs
-        costs = (grid_import * grid_price) - (grid_export * self.feed_in_tariff) + \
-                (actual_bought * peer_price) - (actual_sold * peer_price)
-        
-        # Calculate rewards
-        final_rewards = self._compute_rewards(
-            grid_import=grid_import, grid_export=grid_export,
-            actual_sold=actual_sold, actual_bought=actual_bought,
-            charge_amount=charge_amount, discharge_amount=discharge_amount,
-            costs=costs, grid_price=grid_price, peer_price=peer_price
-        )
-
-        # Metric calculations for the current step
-        no_p2p_import_this_step = np.array([
-            self.no_p2p_import_day[hid][self.current_step] for hid in self.house_ids
-        ], dtype=np.float32)
-
-        # Grid Reduction metrics
-        step_grid_reduction = np.sum(no_p2p_import_this_step - grid_import)
-        self.cumulative_grid_reduction += step_grid_reduction
-        self.grid_reduction_timeseries.append(step_grid_reduction)
-
-        # Check if current grid price corresponds to peak hour
-        if grid_price >= self.max_grid_price * 0.99:
-            self.cumulative_grid_reduction_peak += step_grid_reduction
-
-        # Cost Savings
-        cost_no_p2p = no_p2p_import_this_step * grid_price
-        step_cost_savings_per_agent = cost_no_p2p - costs
-        self.agent_cost_savings += step_cost_savings_per_agent
-        self.cost_savings_timeseries.append(np.sum(step_cost_savings_per_agent))
-
-        # Battery Degradation Cost
-        step_degradation_cost = 0.0
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                batt = self.batteries[hid]
-                degradation_cost_agent = (charge_amount[i] + discharge_amount[i]) * batt["degradation_cost_per_kwh"]
-                step_degradation_cost += degradation_cost_agent
-                
-        self.cumulative_degradation_cost += step_degradation_cost
-        self.degradation_cost_timeseries.append(step_degradation_cost)
-
-        info = {
-            "p2p_buy": actual_bought,
-            "p2p_sell": actual_sold,
-            "grid_import_with_p2p": grid_import,
-            "grid_import_no_p2p": no_p2p_import_this_step,
-            "grid_export": grid_export,
-            "costs": costs,
-            "charge_amount": charge_amount,
-            "discharge_amount": discharge_amount,
-            "step": self.current_step,
-            "step_grid_reduction": step_grid_reduction,
-            "step_cost_savings": np.sum(step_cost_savings_per_agent),
-            "step_degradation_cost": step_degradation_cost,
-        }
-
-        # Increment step & decide "done"
-        self.current_step += 1
-        done = (self.current_step >= self.num_steps)
-
-        # Return next obs, reward list, done, info
-        obs_next = self._get_obs()
-        obs_next_list = [obs_next[i] for i in range(self.num_agents)]
-        rewards_list = [final_rewards[i] for i in range(self.num_agents)]
-
-        return obs_next_list, rewards_list, done, info
-    
-
-    def _get_obs(self):
-        # Build observation array for each agent, including dynamic peer pricing
-        step = min(self.current_step, self.num_steps - 1)
-
-        # Gather per-agent demand/solar into arrays
-        demands = np.array([self.demands[hid][step] for hid in self.house_ids], dtype=np.float32)
-        solars = np.array([self.solars[hid][step] for hid in self.house_ids], dtype=np.float32)
-
-        # Compute market aggregates for dynamic pricing
-        surplus = np.maximum(solars - demands, 0.0)
-        shortfall = np.maximum(demands - solars, 0.0)
-        total_surplus = float(surplus.sum())
-        total_shortfall = float(shortfall.sum())
-
-        grid_price = self.get_grid_price(step)
-        peer_price = self.get_peer_price(step, total_surplus, total_shortfall)
-
-        # Compute time-of-day feature
-        ts = self.data.index[step]
-        hour = ts.hour + ts.minute / 60.0
-
-        # Build per-agent obs
-        obs = []
-        for i, hid in enumerate(self.house_ids):
-            own_demand = demands[i]
-            own_solar = solars[i]
-
-            # Compute state-of-charge fraction (0–1), -1 for non-battery agents
-            if hid in self.batteries:
-                soc_frac = self.batteries[hid]["soc"] / self.batteries[hid]["max_capacity"]
-            else:
-                soc_frac = -1.0
-
-            obs.append([
-                own_demand,
-                own_solar,
-                soc_frac,
-                grid_price,
-                peer_price,
-                float(demands.sum() - own_demand),
-                float(solars.sum() - own_solar),
-                hour
-            ])
-
-        return np.array(obs, dtype=np.float32)
-
-
-    def _compute_jains_index(self, usage_array):
-        """Simple Jain's Fairness Index."""
-        x = np.array(usage_array, dtype=np.float32)
-        numerator = (np.sum(x))**2
-        denominator = len(x) * np.sum(x**2) + 1e-8
-        return numerator / denominator
-
-
-    def _compute_rewards(
-        self,
-        grid_import,
-        grid_export,
-        actual_sold,
-        actual_bought,
-        charge_amount,
-        discharge_amount,
-        costs,
-        grid_price,
-        peer_price
-    ):
-        # Weights for each component
-        w1 = 0.3; w2 = 0.5; w3 = 0.5; w4 = 0.1; w5 = 0.05; w6 = 0.4; w7 = 1.0
-
-        # Jain's index on total P2P volume
-        jfi = self._compute_jains_index(actual_bought + actual_sold)
-
-        # Normalize prices
-        p_grid_norm = grid_price / self.max_grid_price
-        p_peer_norm = peer_price / self.max_grid_price
-
-        rewards = np.zeros(self.num_agents, dtype=np.float32)
-        for i, hid in enumerate(self.house_ids):
-            # Base reward is negative cost
-            reward = - costs[i] * w7
-
-            # Grid import penalty
-            grid_penalty = w1 * grid_import[i] * p_grid_norm
-
-            # P2P sell & buy bonuses
-            p2p_sell_bonus = w2 * actual_sold[i] * p_peer_norm
-            if peer_price < grid_price:
-                p2p_buy_bonus = w3 * actual_bought[i] * ((grid_price - peer_price) / self.max_grid_price)
-            else:
-                p2p_buy_bonus = 0.0
-
-            # Battery penalties (only solar houses have entries)
-            if hid in self.batteries:
-                batt = self.batteries[hid]
-                soc_frac = batt["soc"] / batt["max_capacity"]
-                soc_penalty = w4 * (soc_frac - 0.5) ** 2
-                degradation_penalty = w5 * (charge_amount[i] + discharge_amount[i]) * batt["degradation_cost_per_kwh"]
-            else:
-                soc_penalty = degradation_penalty = 0.0
-
-            # Fairness
-            fairness_bonus = w6 * jfi
-
-            # Combine
-            reward += (
-                - grid_penalty
-                + p2p_sell_bonus
-                + p2p_buy_bonus
-                - soc_penalty
-                - degradation_penalty
-                + fairness_bonus
-            )
-            rewards[i] = reward
-
-        return rewards
-
-
-    def get_episode_metrics(self):
-        """
-        Return performance metrics for the last completed episode.
-        Call after episode finishes (after env.reset()).
-        """
-        return self.episode_metrics
-
-
-    def save_log(self, filename="env_log.csv"):
-        """Save environment step log to CSV."""
-        columns = [
-            "Step", "Total_Grid_Import", "Total_Grid_Export",
-            "Total_P2P_Buy", "Total_P2P_Sell", "Total_Cost",
-        ]
-        df = pd.DataFrame(self.env_log, columns=columns)
-        df.to_csv(filename, index=False)
-        print(f"Environment log saved to {filename}")

Other_algorithms/HC_MAPPO/Environment/cluster_env_wrapper.py

@@ -1,164 +0,0 @@
-import gym
-import numpy as np
-import math
-import sys
-import os
-import functools
-
-import pandas as pd
-
-# Ensure SolarSys Environement is on the Python path
-# Please ensure you follow proper directory structure for running this code
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-from Environment.solar_sys_environment import SolarSys
-
-
-def form_clusters(metrics: dict, size: int) -> list:
-    """
-    Forms balanced, heterogeneous clusters by categorizing houses based on their
-    energy profile and distributing them evenly in a round-robin fashion.
-    """
-    house_ids = list(metrics.keys())
-    if not house_ids:
-        return []
-    all_consumption = [m['consumption'] for m in metrics.values()]
-    all_solar = [m['solar'] for m in metrics.values()]
-    
-    median_consumption = np.median(all_consumption) if all_consumption else 0
-    median_solar = np.median(all_solar) if all_solar else 0
-
-    #Categorize each house based on its profile relative to the median
-    producers = [h for h in house_ids if metrics[h]['solar'] >= median_solar and metrics[h]['consumption'] < median_consumption]
-    consumers = [h for h in house_ids if metrics[h]['solar'] < median_solar and metrics[h]['consumption'] >= median_consumption]
-    prosumers = [h for h in house_ids if metrics[h]['solar'] >= median_solar and metrics[h]['consumption'] >= median_consumption]
-    neutrals = [h for h in house_ids if metrics[h]['solar'] < median_solar and metrics[h]['consumption'] < median_consumption]
-
-    # Create a master list ordered by category
-    sorted_categorized_houses = producers + consumers + prosumers + neutrals
-    
-    # Add any houses that weren't categorized to ensure none are missed
-    categorized_set = set(sorted_categorized_houses)
-    uncategorized = [h for h in house_ids if h not in categorized_set]
-    final_house_list = sorted_categorized_houses + uncategorized
-    num_houses = len(house_ids)
-    num_clusters = math.ceil(num_houses / size)
-    
-    clusters = [[] for _ in range(num_clusters)]
-    
-    for i, house_id in enumerate(final_house_list):
-        target_cluster_idx = i % num_clusters
-        clusters[target_cluster_idx].append(house_id)
-
-    return clusters
-
-class GlobalPriceVecEnvWrapper(gym.vector.VectorEnvWrapper):
-    def __init__(self, env, clusters: list):
-        super().__init__(env)
-        self.clusters = clusters
-        # Expose the underlying SolarSys environments for inspection by the coordinator
-        # self.env.envs gets the list of individual envs from the SyncVectorEnv
-        self.cluster_envs = self.env.envs
-
-    def step(self, actions: np.ndarray, exports: np.ndarray = None, imports: np.ndarray = None):
-        num_clusters = len(self.cluster_envs)
-        net_transfers = np.zeros(num_clusters)
-        if exports is not None and imports is not None:
-            net_transfers = imports - exports
-        batched_low_level_actions = actions
-        batched_transfers = net_transfers.reshape(-1, 1).astype(np.float32)
-        batched_prices = np.full((num_clusters, 1), -1.0, dtype=np.float32)
-        final_packed_actions_tuple = (batched_low_level_actions, batched_transfers, batched_prices)
-        obs_next, rewards, terminateds, truncateds, infos = self.env.step(final_packed_actions_tuple)
-        dones = terminateds | truncateds
-        done_all = dones.all()
-
-
-
-        if done_all:
-            final_infos = infos['final_info']
-            keys = final_infos[0].keys()
-            infos = {k: np.stack([info[k] for info in final_infos]) for k in keys}
-
-        info_agg = {
-            "cluster_dones": dones,
-            "cluster_infos": infos,
-        }
-        
-        return obs_next, rewards, done_all, info_agg
-
-    def get_export_capacity(self, cluster_idx: int) -> float:
-        """Returns the total physically exportable energy from a cluster's batteries and solar in kWh."""
-        cluster_env = self.cluster_envs[cluster_idx]
-        available_from_batt = cluster_env.battery_soc * cluster_env.battery_discharge_efficiency
-        total_exportable = np.sum(available_from_batt) + cluster_env.current_solar
-        return float(total_exportable)
-
-    def get_import_capacity(self, cluster_idx: int) -> float:
-        """Returns the total physically importable space in a cluster's batteries in kWh."""
-        cluster_env = self.cluster_envs[cluster_idx]
-        free_space = cluster_env.battery_max_capacity - cluster_env.battery_soc
-        total_storable = np.sum(free_space)
-        return float(total_storable)
-
-    def send_energy(self, from_cluster_idx: int, amount: float) -> float:
-        """Drains 'amount' of energy from the specified cluster (batteries first, then solar)."""
-        cluster_env = self.cluster_envs[from_cluster_idx]
-        return cluster_env.send_energy(amount)
-
-    def receive_energy(self, to_cluster_idx: int, amount: float) -> float:
-        """Charges batteries in the specified cluster with 'amount' of energy."""
-        cluster_env = self.cluster_envs[to_cluster_idx]
-        return cluster_env.receive_energy(amount)
-
-
-def make_vec_env(data_path: str, time_freq: str, cluster_size: int, state: str):
-    print("--- Pre-loading shared dataset for all environments ---")
-    try:
-        shared_df = pd.read_csv(data_path)
-        shared_df["local_15min"] = pd.to_datetime(shared_df["local_15min"], utc=True)
-        shared_df.set_index("local_15min", inplace=True)
-
-        #  ADD THIS LINE 
-        shared_df = shared_df.resample(time_freq).mean()
-        #   ADD THIS LINE 
-
-    except Exception as e:
-        raise ValueError(f"Failed to pre-load data in make_vec_env: {e}")
-
-    base_env_for_metrics = SolarSys(
-        data_path=data_path,
-        time_freq=time_freq,
-        preloaded_data=shared_df,  # Pass the shared DataFrame here
-        state=state
-    )
-    
-    # This part for calculating metrics and forming clusters
-    metrics = {}
-    for hid in base_env_for_metrics.house_ids:
-        total_consumption = float(
-            np.clip(base_env_for_metrics.original_no_p2p_import[hid], 0.0, None).sum()
-        )
-        total_solar = float(
-            base_env_for_metrics.all_data[f"total_solar_{hid}"].clip(lower=0.0).sum()
-        )
-        metrics[hid] = {'consumption': total_consumption, 'solar': total_solar}
-    
-    clusters = form_clusters(metrics, cluster_size)
-    print(f"Formed {len(clusters)} clusters of size up to {cluster_size}.")
-
-    # functools.partial to create environment
-    env_fns = []
-    for cluster_house_ids in clusters:
-        preset_env_fn = functools.partial(
-            SolarSys,
-            data_path=data_path,
-            time_freq=time_freq,
-            house_ids_in_cluster=cluster_house_ids,
-            preloaded_data=shared_df,
-            state=state 
-        )
-        env_fns.append(preset_env_fn)
-    sync_vec_env = gym.vector.SyncVectorEnv(env_fns)
-    wrapped_vec_env = GlobalPriceVecEnvWrapper(sync_vec_env, clusters=clusters)
-
-    return wrapped_vec_env

Other_algorithms/HC_MAPPO/Environment/solar_sys_environment.py

@@ -1,673 +0,0 @@
-import gym
-import pandas as pd
-import numpy as np
-from collections import deque
-import random
-from gym.spaces import Tuple, Box
-
-random.seed(42)
-np.random.seed(42)
-
-class SolarSys(gym.Env):
-
-    def __init__(
-        self,
-        data_path="DATA/training/25houses_152days_TRAIN.csv",
-        state="", # Select from 'oklahoma', 'colorado', 'pennsylvania'
-        time_freq="15T",  
-        house_ids_in_cluster=None,
-        preloaded_data=None
-        
-    ):
-        
-        super().__init__()  # initialize parent gym.Env
-        self.state = state.lower()
-
-        # --- Centralized Pricing Configuration ---
-        self._pricing_info = {
-            "oklahoma": {
-                "max_grid_price": 0.2112,
-                "feed_in_tariff": 0.04,
-                "price_function": self._get_oklahoma_price
-            },
-            "colorado": {
-                "max_grid_price": 0.32,
-                "feed_in_tariff": 0.055,
-                "price_function": self._get_colorado_price
-            },
-            "pennsylvania": {
-                "max_grid_price": 0.5505,
-                "feed_in_tariff": 0.06,
-                "price_function": self._get_pennsylvania_price
-            }
-        }
-
-        if self.state not in self._pricing_info:
-            raise ValueError(f"State '{self.state}' is not supported. Available states: {list(self._pricing_info.keys())}")
-
-        state_config = self._pricing_info[self.state]
-        self.max_grid_price = state_config["max_grid_price"]
-        self.feed_in_tariff = state_config["feed_in_tariff"]
-        self._get_price_function = state_config["price_function"]       
-        self.data_path      = data_path
-        self.time_freq      = time_freq
-        if preloaded_data is not None:
-            all_data = preloaded_data
-            if house_ids_in_cluster:
-                 print(f"Using pre-loaded data for cluster with {len(house_ids_in_cluster)} houses.")
-        else:
-            print(f"Loading data from {data_path}...")
-            try:
-                all_data = pd.read_csv(data_path)
-                all_data["local_15min"] = pd.to_datetime(all_data["local_15min"], utc=True)
-                all_data.set_index("local_15min", inplace=True)
-
-            except FileNotFoundError:
-                raise FileNotFoundError(f"Data file {data_path} not found.")
-            except pd.errors.EmptyDataError:
-                raise ValueError(f"Data file {data_path} is empty.")
-            except Exception as e:
-                raise ValueError(f"Error loading data: {e}")
-
-
-        # Compute global maxima for normalization
-        grid_cols  = [c for c in all_data.columns if c.startswith("grid_")]
-        solar_cols = [c for c in all_data.columns if c.startswith("total_solar_")]
-        all_grid  = all_data[grid_cols].values       
-        all_solar = all_data[solar_cols].values     
-
-        # max total demand = max(grid + solar) over all time & agents
-        self.global_max_demand = float((all_grid + all_solar).max()) + 1e-8
-
-        # max solar generation alone
-        self.global_max_solar  = float(all_solar.max()) + 1e-8
-
-        # Store the resampled dataset
-        self.all_data = all_data
-        all_house_ids_in_file = [
-            col.split("_")[1] for col in self.all_data.columns
-            if col.startswith("grid_")
-        ]
-        if house_ids_in_cluster:
-            self.house_ids = [hid for hid in house_ids_in_cluster if hid in all_house_ids_in_file]
-        else:
-            self.house_ids = all_house_ids_in_file
-        
-        if not self.house_ids:
-            raise ValueError("No valid house_ids found for this environment instance.")
-        
-        self.env_log_infos = []  
-
-        self.time_freq = time_freq 
-        freq_offset = pd.tseries.frequencies.to_offset(time_freq)
-        minutes_per_step = freq_offset.nanos / 1e9 / 60.0
-        self.steps_per_day = int(24 * 60 // minutes_per_step)
-
-        total_rows = len(self.all_data)
-        self.total_days = total_rows // self.steps_per_day
-        if self.total_days < 1:
-            raise ValueError(
-                f"After resampling, dataset has {total_rows} rows, which is "
-                f"less than a single day of {self.steps_per_day} steps."
-            )
-
-        self.num_agents = len(self.house_ids)
-        self.original_no_p2p_import = {}
-        for hid in self.house_ids:
-            col_grid = f"grid_{hid}"
-            self.original_no_p2p_import[hid] = self.all_data[col_grid].clip(lower=0.0).values
-        solar_cols = [f"total_solar_{hid}" for hid in self.house_ids]
-        solar_sums = self.all_data[solar_cols].sum(axis=0).to_dict()
-        self.agent_groups = [
-            1 if solar_sums[f"total_solar_{hid}"] > 0 else 0
-            for hid in self.house_ids
-        ]
-
-        self.group_counts = {
-            0: self.agent_groups.count(0),
-            1: self.agent_groups.count(1)
-        }
-        print(f"Number of houses in each group: {self.group_counts}")
-
-        #battery logic
-        self.battery_options = {
-            "teslapowerwall": {"max_capacity": 13.5, "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 5.0, "max_discharge_rate": 5.0, "degradation_cost_per_kwh": 0.005},
-            "enphase":         {"max_capacity": 5.0,  "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 2.0, "max_discharge_rate": 2.0, "degradation_cost_per_kwh": 0.005},
-            "franklin":        {"max_capacity": 15.0, "charge_efficiency": 0.95, "discharge_efficiency": 0.90, "max_charge_rate": 6.0, "max_discharge_rate": 6.0, "degradation_cost_per_kwh": 0.005},
-        }
-        self.solar_houses = [
-            hid for hid in self.house_ids
-            if (self.all_data[f"total_solar_{hid}"] > 0).any()
-        ]
-      
-        self.batteries = {}
-        for hid in self.solar_houses:
-            choice = random.choice(list(self.battery_options))
-            specs  = self.battery_options[choice]
-            self.batteries[hid] = {"soc": 0.0, **specs}
-
-        self.battery_charge_history = {hid: [] for hid in self.batteries}
-        self.battery_discharge_history = {hid: [] for hid in self.batteries}
-        self.battery_capacity = sum(b["max_capacity"] for b in self.batteries.values())
-        self.battery_level    = sum(b["soc"]          for b in self.batteries.values())
-        self.current_solar    = 0.0
-        self.has_battery = np.array([1 if hid in self.batteries else 0 for hid in self.house_ids], dtype=np.float32)
-
-        # Initialize arrays for all agents, with zeros for non-battery agents
-        self.battery_soc = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_max_capacity = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_charge_efficiency = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_discharge_efficiency = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_max_charge_rate = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_max_discharge_rate = np.zeros(self.num_agents, dtype=np.float32)
-        self.battery_degradation_cost = np.zeros(self.num_agents, dtype=np.float32)
-
-        # Populate the arrays using the created battery dictionary
-        for i, hid in enumerate(self.house_ids):
-            if hid in self.batteries:
-                batt = self.batteries[hid]
-                self.battery_max_capacity[i] = batt["max_capacity"]
-                self.battery_charge_efficiency[i] = batt["charge_efficiency"]
-                self.battery_discharge_efficiency[i] = batt["discharge_efficiency"]
-                self.battery_max_charge_rate[i] = batt["max_charge_rate"]
-                self.battery_max_discharge_rate[i] = batt["max_discharge_rate"]
-                self.battery_degradation_cost[i] = batt["degradation_cost_per_kwh"]
-
-
-        # ========== SPACES (Observation & Action) ===================================
-        self.observation_space = gym.spaces.Box(
-            low=-np.inf, high=np.inf,
-            shape=(self.num_agents, 8),
-            dtype=np.float32
-        )
-        self.action_space = Tuple((
-            Box(low=0.0, high=1.0, shape=(self.num_agents, 6), dtype=np.float32),
-            Box(low=-np.inf, high=np.inf, shape=(1,), dtype=np.float32),
-            Box(low=-1.0, high=np.inf, shape=(1,), dtype=np.float32)
-        ))
-        
-        # ========== REWARD FUNCTION PARAMETERS ======================================
-        self.data = None
-        self.env_log = []
-        self.day_index = -1  
-        self.current_step = 0
-        self.num_steps = self.steps_per_day  
-        self.demands = {}
-        self.solars = {}
-        self.previous_actions = {
-            hid: np.zeros(6) for hid in self.house_ids
-        }
-        self._initialize_episode_metrics()
-
-    def get_grid_price(self, step_idx):
-        """
-        Returns the grid price for the current step based on the selected state.
-        """
-        return self._get_price_function(step_idx)
-
-    def _get_oklahoma_price(self, step_idx):
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        if 14 <= hour < 19:
-            return 0.2112
-        else:
-            return 0.0434
-
-    def _get_colorado_price(self, step_idx):
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        if 15 <= hour < 19:
-            return 0.32
-        elif 13 <= hour < 15:
-            return 0.22
-        else:
-            return 0.12
-
-    def _get_pennsylvania_price(self, step_idx):
-        minutes_per_step = 24 * 60 / self.steps_per_day
-        hour = int((step_idx * minutes_per_step) // 60) % 24
-        if 13 <= hour < 21:
-            return 0.125048
-        elif hour >= 23 or hour < 6:
-            return 0.057014
-        else:
-            return 0.079085
-        
-    def get_peer_price(self, step_idx, total_surplus, total_shortfall):
-        grid_price = self.get_grid_price(step_idx)
-        feed_in_tariff = self.feed_in_tariff
-        
-        # Parameters for arctangent-log pricing
-        p_balance = (grid_price * 0.80) + (feed_in_tariff * 0.20)
-        p_con = (grid_price - feed_in_tariff) / (1.5 * np.pi)
-        k = 1.5 
-        epsilon = 1e-6
-        supply = total_surplus + epsilon
-        demand = total_shortfall + epsilon
-        
-        ratio = demand / supply
-        log_ratio = np.log(ratio)
-        if log_ratio < 0:
-            power_term = - (np.abs(log_ratio) ** k)
-        else:
-            power_term = log_ratio ** k
-        
-        price_offset = 2 * np.pi * p_con * np.arctan(power_term)
-        
-        peer_price = p_balance + price_offset
-        
-        final_price = float(np.clip(peer_price, feed_in_tariff, grid_price))
-        
-        return final_price
-
- 
-    def _initialize_episode_metrics(self):
-        """Initializes or resets all metrics tracked over a single episode (day)."""
-        self.cumulative_grid_reduction = 0.0
-        self.cumulative_grid_reduction_peak = 0.0
-        self.cumulative_degradation_cost = 0.0
-        self.agent_cost_savings = np.zeros(self.num_agents)
-        self.degradation_cost_timeseries = []
-        self.cost_savings_timeseries = []
-        self.grid_reduction_timeseries = []
-
-    def get_episode_metrics(self):
-        """
-        Returns a dictionary of performance metrics for the last completed episode.
-        """
-        return self.episode_metrics
-        
-   ##########################################################################
-    # Gym Required Methods
- 
-    def reset(self):
-        if self.current_step > 0:
-            positive_savings = self.agent_cost_savings[self.agent_cost_savings > 0]
-            if len(positive_savings) > 1:
-                fairness_on_savings = self._compute_jains_index(positive_savings)
-            else:
-                fairness_on_savings = 0.0
-
-            self.episode_metrics = {
-                "grid_reduction_entire_day": self.cumulative_grid_reduction,
-                "grid_reduction_peak_hours": self.cumulative_grid_reduction_peak,
-                "total_cost_savings": np.sum(self.agent_cost_savings),
-                "fairness_on_cost_savings": fairness_on_savings,
-                "battery_degradation_cost_total": self.cumulative_degradation_cost,
-                "degradation_cost_over_time": self.degradation_cost_timeseries,
-                "cost_savings_over_time": self.cost_savings_timeseries,
-                "grid_reduction_over_time": self.grid_reduction_timeseries,
-            }
-        self.day_index = np.random.randint(0, self.total_days)
-
-        start_row = self.day_index * self.steps_per_day
-        end_row = start_row + self.steps_per_day
-        day_data = self.all_data.iloc[start_row:end_row].copy()
-        self.data = day_data  
-
-        self.no_p2p_import_day = {}
-        for hid in self.house_ids:
-            self.no_p2p_import_day[hid] = self.original_no_p2p_import[hid][start_row:end_row]
-
-        demand_list = []
-        solar_list = []
-        for hid in self.house_ids:
-            col_grid = f"grid_{hid}"
-            col_solar = f"total_solar_{hid}"
-
-            grid_series = day_data[col_grid].fillna(0.0)
-            solar_series = day_data[col_solar].fillna(0.0).clip(lower=0.0)
-
-            demand_array = grid_series.values + solar_series.values
-            demand_array = np.clip(demand_array, 0.0, None)
-
-            demand_list.append(demand_array)
-            solar_list.append(solar_series.values)
-
-        self.demands_day = np.stack(demand_list, axis=1).astype(np.float32)
-        self.solars_day = np.stack(solar_list, axis=1).astype(np.float32)
-
-        self.hours_day = (self.data.index.hour + self.data.index.minute / 60.0).values
-
-        self.current_step = 0
-        self.env_log     = []
-        for hid in self.house_ids:
-            self.previous_actions[hid] = np.zeros(6)
-
-        lows = 0.30 * self.battery_max_capacity
-        highs = 0.70 * self.battery_max_capacity
-
-        self.battery_soc = np.random.uniform(low=lows, high=highs)
-        self.battery_soc *= self.has_battery
-
-        initial_demands = self.demands_day[0]
-        initial_solars = self.solars_day[0]
-        initial_surplus = np.maximum(initial_solars - initial_demands, 0.0).sum()
-        initial_shortfall = np.maximum(initial_demands - initial_solars, 0.0).sum()
-        initial_peer_price = self.get_peer_price(0, initial_surplus, initial_shortfall)
-
-        obs = self._get_obs(peer_price=initial_peer_price)
-
-        self._initialize_episode_metrics()
-
-        return obs, {}
-
-    def step(self, packed_action):
-        actions, transfer_kwh_arr, peer_price_arr = packed_action
-        inter_cluster_transfer_kwh = float(transfer_kwh_arr[0])
-        override_peer_price_val = float(peer_price_arr[0])
-        
-        override_peer_price = override_peer_price_val if override_peer_price_val >= 0 else None
-
-        actions = np.array(actions, dtype=np.float32)
-        if actions.shape != (self.num_agents, 6):
-            raise ValueError(f"Actions shape mismatch: got {actions.shape}, expected {(self.num_agents, 6)}")
-        actions = np.clip(actions, 0.0, 1.0)
-
-        a_sellGrid      = actions[:, 0]
-        a_buyGrid       = actions[:, 1]
-        a_sellPeers     = actions[:, 2]
-        a_buyPeers      = actions[:, 3]
-        a_chargeBatt    = actions[:, 4]
-        a_dischargeBatt = actions[:, 5]
-        
-
-        demands = self.demands_day[self.current_step]
-        solars  = self.solars_day[self.current_step]
-
-        total_surplus   = np.maximum(solars  - demands, 0.0).sum()
-        total_shortfall = np.maximum(demands - solars,  0.0).sum()
-        self.current_solar = total_surplus
-
-        if override_peer_price is not None:
-            peer_price = override_peer_price
-        else:
-            peer_price = self.get_peer_price(
-                self.current_step,
-                total_surplus,
-                total_shortfall
-            )
-        
-        grid_price = self.get_grid_price(self.current_step)
-    
-        shortfall = np.maximum(demands - solars, 0.0)
-        surplus   = np.maximum(solars  - demands, 0.0)
-        
-        final_shortfall = shortfall.copy()
-        final_surplus   = surplus.copy()
-        grid_import     = np.zeros(self.num_agents, dtype=np.float32)
-        grid_export     = np.zeros(self.num_agents, dtype=np.float32)
-
-        # ### VECTORIZED BATTERY DISCHARGE ###
-        available_from_batt = self.battery_soc * self.battery_discharge_efficiency
-        desired_discharge = a_dischargeBatt * self.battery_max_discharge_rate
-        discharge_amount = np.minimum.reduce([desired_discharge, available_from_batt, final_shortfall])
-        discharge_amount *= self.has_battery # Ensure only batteries discharge
-
-        # Update SOC (energy drawn from battery before efficiency loss)
-        self.battery_soc -= (discharge_amount / (self.battery_discharge_efficiency + 1e-9)) * self.has_battery
-        self.battery_soc = np.maximum(0.0, self.battery_soc)
-        final_shortfall -= discharge_amount
-
-        cap_left = self.battery_max_capacity - self.battery_soc
-        desired_charge = a_chargeBatt * self.battery_max_charge_rate
-        charge_amount = np.minimum.reduce([
-            desired_charge,
-            cap_left / (self.battery_charge_efficiency + 1e-9),
-            final_surplus
-        ])
-        charge_amount *= self.has_battery 
-
-        # Update SOC 
-        self.battery_soc += charge_amount * self.battery_charge_efficiency
-        final_surplus -= charge_amount
-
-        
-
-        # ### VECTORIZED P2P TRADING ###
-        battery_offer = (self.battery_soc * self.battery_discharge_efficiency) * self.has_battery
-        effective_surplus = final_surplus + battery_offer
-
-        netPeer = a_buyPeers - a_sellPeers
-        p2p_buy_request = np.maximum(0, netPeer) * final_shortfall
-        p2p_sell_offer = np.maximum(0, -netPeer) * effective_surplus
-
-        total_sell = np.sum(p2p_sell_offer)
-        total_buy  = np.sum(p2p_buy_request)
-        matched    = min(total_sell, total_buy)
-
-        if matched > 1e-9:
-            sell_fraction = p2p_sell_offer / (total_sell + 1e-12)
-            buy_fraction  = p2p_buy_request / ( total_buy + 1e-12)
-            actual_sold   = matched * sell_fraction
-            actual_bought = matched * buy_fraction
-        else:
-            actual_sold   = np.zeros(self.num_agents, dtype=np.float32)
-            actual_bought = np.zeros(self.num_agents, dtype=np.float32)
-        
-
-        from_batt = np.minimum(actual_sold, battery_offer)
-        from_solar = actual_sold - from_batt
-
-        final_surplus -= from_solar
-        
-        final_shortfall -= actual_bought
-        soc_reduction = (from_batt / (self.battery_discharge_efficiency + 1e-9)) * self.has_battery
-        self.battery_soc -= soc_reduction
-        self.battery_soc = np.maximum(0.0, self.battery_soc) 
-
-
-        if inter_cluster_transfer_kwh > 0:
-            amount_received = inter_cluster_transfer_kwh
-            
-           
-            total_shortfall_in_cluster = np.sum(final_shortfall)
-            if total_shortfall_in_cluster > 1e-6:
-                
-                to_cover_shortfall = min(amount_received, total_shortfall_in_cluster)
-                distribution_ratio = final_shortfall / total_shortfall_in_cluster
-                shortfall_reduction = distribution_ratio * to_cover_shortfall
-                final_shortfall -= shortfall_reduction
-                
-                amount_received -= to_cover_shortfall
-
-            if amount_received > 1e-6:
-           
-                cap_left = self.battery_max_capacity - self.battery_soc
-                storable_energy = cap_left / (self.battery_charge_efficiency + 1e-9)
-                total_storable_in_cluster = np.sum(storable_energy * self.has_battery)
-
-                if total_storable_in_cluster > 1e-6:
-                    
-                    to_store = min(amount_received, total_storable_in_cluster)
-                    
-                    
-                    storage_ratio = storable_energy / total_storable_in_cluster
-                    energy_to_store_per_batt = storage_ratio * to_store
-                    
-                    
-                    self.battery_soc += (energy_to_store_per_batt * self.battery_charge_efficiency) * self.has_battery
-
-        elif inter_cluster_transfer_kwh < 0:  
-            amount_to_send = abs(inter_cluster_transfer_kwh)
-            
-            
-            total_surplus_in_cluster = np.sum(final_surplus)
-            if total_surplus_in_cluster > 1e-6:
-                
-                sent_from_surplus = min(amount_to_send, total_surplus_in_cluster)           
-                draw_ratio = final_surplus / total_surplus_in_cluster
-                surplus_reduction = draw_ratio * sent_from_surplus
-                final_surplus -= surplus_reduction   
-                amount_to_send -= sent_from_surplus
-
-            
-            if amount_to_send > 1e-6:
-                
-                available_from_batt = (self.battery_soc * self.battery_discharge_efficiency) * self.has_battery
-                total_available_from_batt = np.sum(available_from_batt)
-
-                if total_available_from_batt > 1e-6:
-                    # Discharge a maximum of 'amount_to_send' from batteries
-                    to_discharge = min(amount_to_send, total_available_from_batt)
-                    
-                    # Draw this amount proportionally from each available battery
-                    discharge_ratio = available_from_batt / total_available_from_batt
-                    discharged_per_batt = discharge_ratio * to_discharge # This is effective energy
-                    
-                    # Update SoC (energy drawn from battery before efficiency loss)
-                    soc_reduction = (discharged_per_batt / (self.battery_discharge_efficiency + 1e-9))
-                    self.battery_soc -= soc_reduction * self.has_battery
-                    self.battery_soc = np.maximum(0.0, self.battery_soc)
-        # =======================================================================
-        
-        netGrid = a_buyGrid - a_sellGrid
-        grid_import = np.maximum(0, netGrid) * final_shortfall
-        grid_export = np.maximum(0, -netGrid) * final_surplus
-
-        forced = np.maximum(final_shortfall - grid_import, 0.0)
-        grid_import += forced
-        final_shortfall -= forced       
-
-        feed_in_tariff = self.feed_in_tariff
-        costs = (
-            (grid_import * grid_price)
-            - (grid_export * feed_in_tariff)
-            + (actual_bought * peer_price)
-            - (actual_sold * peer_price)
-        )
-       
-        final_rewards = self._compute_rewards(
-            grid_import=grid_import, grid_export=grid_export, actual_sold=actual_sold,
-            actual_bought=actual_bought, charge_amount=charge_amount, discharge_amount=discharge_amount,
-            costs=costs, grid_price=grid_price, peer_price=peer_price
-        )
-
-        no_p2p_import_this_step = np.array([
-            self.no_p2p_import_day[hid][self.current_step] 
-            for hid in self.house_ids
-        ], dtype=np.float32)
-
-
-        # --- Metric 1 & 2: Grid Reduction (Entire Day & Peak Hours) ---
-        step_grid_reduction = np.sum(no_p2p_import_this_step - grid_import)
-        self.cumulative_grid_reduction += step_grid_reduction
-        self.grid_reduction_timeseries.append(step_grid_reduction)
-
-        if grid_price >= self.max_grid_price * 0.99:
-            self.cumulative_grid_reduction_peak += step_grid_reduction
-
-        # --- Metric 3: Total Cost Savings ---
-        cost_no_p2p = no_p2p_import_this_step * grid_price
-        step_cost_savings_per_agent = cost_no_p2p - costs
-        self.agent_cost_savings += step_cost_savings_per_agent
-        self.cost_savings_timeseries.append(np.sum(step_cost_savings_per_agent))
-
-        # --- Metric 5 & 6: Battery Degradation Cost (Total and Over Time) ---
-        degradation_cost_agent = (charge_amount + discharge_amount) * self.battery_degradation_cost
-        step_degradation_cost = np.sum(degradation_cost_agent)
-
-        self.cumulative_degradation_cost += step_degradation_cost
-        self.degradation_cost_timeseries.append(step_degradation_cost)
-
-        info = {
-            "p2p_buy": actual_bought,
-            "p2p_sell": actual_sold,
-            "grid_import_with_p2p": grid_import,
-            "grid_import_no_p2p": no_p2p_import_this_step,
-            "grid_export": grid_export,
-            "costs": costs,
-            "charge_amount": charge_amount,
-            "discharge_amount": discharge_amount,
-            "step": self.current_step,
-            "step_grid_reduction": step_grid_reduction,
-            "step_cost_savings": np.sum(step_cost_savings_per_agent),
-            "step_degradation_cost": step_degradation_cost,
-        }
-
-        self.env_log.append([
-            self.current_step, np.sum(grid_import), np.sum(grid_export),
-            np.sum(actual_bought), np.sum(actual_sold), np.sum(costs)
-        ])
-
-        self.current_step += 1
-       
-        terminated = False 
-        truncated = (self.current_step >= self.num_steps)
-
-        obs_next = self._get_obs(peer_price=peer_price)
-        info['agent_rewards'] = final_rewards
-        self.last_info = info
-        self.env_log_infos.append(info)
-        return obs_next, final_rewards.sum(), terminated, truncated, info
-        
-    
-
-    def _get_obs(self, peer_price: float):
-        step = min(self.current_step, self.num_steps - 1)
-        demands = self.demands_day[step]
-        solars  = self.solars_day[step]
-        grid_price = self.get_grid_price(step)
-        hour = self.hours_day[step]
-        soc_frac = self.battery_soc / (self.battery_max_capacity + 1e-9)
-        soc_frac = np.where(self.has_battery == 1, soc_frac, -1.0)
-        total_demand_others = demands.sum() - demands
-        total_solar_others = solars.sum() - solars
-
-        obs = np.stack([
-            demands,
-            solars,
-            soc_frac,
-            np.full(self.num_agents, grid_price),
-            np.full(self.num_agents, peer_price),
-            total_demand_others,
-            total_solar_others,
-            np.full(self.num_agents, hour)
-        ], axis=1).astype(np.float32)
-
-        return obs
-
-
-    def _compute_jains_index(self, usage_array):
-        x = np.array(usage_array, dtype=np.float32)
-        numerator = (np.sum(x))**2
-        denominator = len(x) * np.sum(x**2) + 1e-8
-        return numerator / denominator
-
-
-    def _compute_rewards(
-        self, grid_import, grid_export, actual_sold, actual_bought,
-        charge_amount, discharge_amount, costs, grid_price, peer_price
-    ):
-        
-        w1 = 0.3; w2 = 0.5; w3 = 0.5; w4 = 0.1; w5 = 0.05; w6 = 0.4; w7 = 1.0
-
-        p_grid_norm = grid_price / self.max_grid_price
-        p_peer_norm = peer_price / self.max_grid_price
-
-        rewards = -costs * w7
-        rewards -= w1 * grid_import * p_grid_norm
-        rewards += w2 * actual_sold * p_peer_norm
-        buy_bonus = w3 * actual_bought * ((grid_price - peer_price) / self.max_grid_price)
-        rewards += np.where(peer_price < grid_price, buy_bonus, 0.0)
-
-        # ### VECTORIZED REWARD PENALTIES ###
-        soc_frac = self.battery_soc / (self.battery_max_capacity + 1e-9)
-        soc_penalties = w4 * ((soc_frac - 0.5) ** 2) * self.has_battery
-        degrad_penalties = w5 * (charge_amount + discharge_amount) * self.battery_degradation_cost
-
-        rewards -= soc_penalties
-        rewards -= degrad_penalties
-
-        jfi = self._compute_jains_index(actual_bought + actual_sold)
-        rewards += w6 * jfi
-        return rewards
-        
-    def save_log(self, filename="env_log.csv"):
-        columns = [
-            "Step", "Total_Grid_Import", "Total_Grid_Export",
-            "Total_P2P_Buy", "Total_P2P_Sell", "Total_Cost",
-        ]
-        df = pd.DataFrame(self.env_log, columns=columns)
-        df.to_csv(filename, index=False)
-        print(f"Environment log saved to {filename}")

Other_algorithms/HC_MAPPO/HC_MAPPO_evaluation.py

@@ -1,618 +0,0 @@
-import os
-import sys
-import time
-from datetime import datetime
-import re
-import numpy as np
-import torch
-import pandas as pd
-import matplotlib.pyplot as plt
-import glob
-
-# Allow imports from project root
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from cluster import InterClusterCoordinator, InterClusterLedger
-from Environment.cluster_env_wrapper import make_vec_env 
-from mappo.trainer.mappo import MAPPO
-
-
-def compute_jains_fairness(values: np.ndarray) -> float:
-    """
-    Compute Jain's fairness index for a given array of values.
-    Returns a value between 0 and 1, where 1 indicates perfect fairness.
-    """
-    if len(values) == 0:
-        return 0.0
-    if np.all(values == 0):
-        return 1.0
-    num = (values.sum())**2
-    den = len(values) * (values**2).sum() + 1e-8
-    return float(num / den)
-
-
-def main():
-    # Configuration Parameters
-    DATA_PATH = "data/testing/500houses_30days_TEST.csv"
-    MODEL_DIR = "models/hierarchical_oklahoma_500agents_10size_10000eps_latest/models"
-    
-    # Auto-detect state from model path
-    state_match = re.search(r"hierarchical_(oklahoma|colorado|pennsylvania)_", MODEL_DIR)
-    if not state_match:
-        # Fallback to searching the parent directory name if the first pattern fails
-        state_match = re.search(r"mappo_(oklahoma|colorado|pennsylvania)_", MODEL_DIR)
-
-    if not state_match:
-        raise ValueError(
-            "Could not automatically detect the state (oklahoma, colorado, or pennsylvania) "
-            "from the model directory path. Please ensure the path contains the state name."
-        )
-    detected_state = state_match.group(1)
-    print(f"--- Detected state: {detected_state.upper()} ---")
-
-    # Auto-detect cluster size from model path
-    cluster_size_match = re.search(r'(\d+)size_', MODEL_DIR)
-    if not cluster_size_match:
-        raise ValueError(
-            "Could not automatically detect the cluster size from the model directory path. "
-            "Please ensure the path contains a pattern like '5size_' or '10size_'."
-        )
-    detected_cluster_size = int(cluster_size_match.group(1))
-    print(f"--- Detected cluster size: {detected_cluster_size} ---")
-    
-    DAYS_TO_EVALUATE = 30
-    SOLAR_THRESHOLD = 0.1
-    MAX_TRANSFER_KWH = 1000000.0
-
-    W_COST_SAVINGS = 1.0
-    W_GRID_PENALTY = 0.5
-    W_P2P_BONUS = 0.2
-
-    # Environment Initialization
-    cluster_env = make_vec_env(
-        data_path=DATA_PATH,
-        time_freq="15T",
-        cluster_size=detected_cluster_size,
-        state=detected_state
-    )
-    n_clusters = cluster_env.num_envs
-    sample_subenv = cluster_env.cluster_envs[0]
-    eval_num_steps = sample_subenv.num_steps
-    print(f"Number of steps per day: {eval_num_steps}")
-
-    # Load intra-cluster MAPPO agents
-    n_agents_per_cluster = sample_subenv.num_agents
-    local_dim = sample_subenv.observation_space.shape[-1]
-    global_dim = n_agents_per_cluster * local_dim
-    act_dim = sample_subenv.action_space[0].shape[-1]
-
-    print(f"Creating and loading {n_clusters} independent low-level MAPPO agents...")
-    low_agents = []
-    for i in range(n_clusters):
-        agent = MAPPO(
-            n_agents=n_agents_per_cluster, 
-            local_dim=local_dim, 
-            global_dim=global_dim, 
-            act_dim=act_dim,
-            lr=2e-4,
-            gamma=0.95, 
-            lam=0.95, 
-            clip_eps=0.2, 
-            k_epochs=4, 
-            batch_size=512, 
-            episode_len=96
-        )
-        ckpt_pattern = os.path.join(MODEL_DIR, f"low_cluster{i}_ep*.pth")
-        ckpts_low = glob.glob(ckpt_pattern)
-        if not ckpts_low:
-            raise FileNotFoundError(f"No checkpoint found for cluster {i} with pattern: {ckpt_pattern}")
-        latest_low = sorted(ckpts_low, key=lambda x: int(re.search(r'ep(\d+)\.pth$', x).group(1)))[-1]
-        print(f"Loading low-level policy for cluster {i} from: {latest_low}")
-        agent.load(latest_low)
-        agent.actor.eval()
-        agent.critic.eval()
-        low_agents.append(agent)
-
-    # Output Folder Setup
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    num_agents = sum(subenv.num_agents for subenv in cluster_env.cluster_envs)
-    run_name = f"eval_vectorized_{num_agents}agents_{DAYS_TO_EVALUATE}days_{timestamp}"
-    output_folder = os.path.join("runs_final_vectorized_eval", run_name)
-    logs_dir = os.path.join(output_folder, "logs")
-    plots_dir = os.path.join(output_folder, "plots")
-    for d in (logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Saving evaluation outputs to: {output_folder}")
-
-    # Load inter-cluster MAPPO agent
-    OBS_DIM_HI_LOCAL = 7
-    act_dim_inter = 2
-    
-    # Define the global dimension for the high-level agent
-    OBS_DIM_HI_GLOBAL = n_clusters * OBS_DIM_HI_LOCAL
-    
-    print(f"Initializing evaluation inter-agent (MAPPO): n_agents={n_clusters}, "
-          f"local_dim={OBS_DIM_HI_LOCAL}, global_dim={OBS_DIM_HI_GLOBAL}, act_dim={act_dim_inter}")
-    
-    # Instantiate MAPPO for inter-cluster coordination
-    inter_agent = MAPPO(
-        n_agents=n_clusters, 
-        local_dim=OBS_DIM_HI_LOCAL, 
-        global_dim=OBS_DIM_HI_GLOBAL, 
-        act_dim=act_dim_inter,
-        lr=2e-4, 
-        gamma=0.95, 
-        lam=0.95, 
-        clip_eps=0.2, 
-        k_epochs=4, 
-        batch_size=512, 
-        episode_len=96
-    )
-    
-    ckpts_inter = glob.glob(os.path.join(MODEL_DIR, "inter_ep*.pth"))
-    if not ckpts_inter:
-        raise FileNotFoundError(f"No high-level checkpoints (inter_ep*.pth) in {MODEL_DIR}")
-    latest_inter = sorted(ckpts_inter, key=lambda x: int(re.search(r'ep(\d+)\.pth$', x).group(1)))[-1]
-    print("Loading inter-cluster policy from", latest_inter)
-    inter_agent.load(latest_inter)
-    inter_agent.actor.eval()
-    inter_agent.critic.eval()
-
-    # Instantiate Coordinator
-    ledger = InterClusterLedger()
-    coordinator = InterClusterCoordinator(
-        cluster_env, 
-        inter_agent, 
-        ledger, 
-        max_transfer_kwh=MAX_TRANSFER_KWH,
-        w_cost_savings=W_COST_SAVINGS, 
-        w_grid_penalty=W_GRID_PENALTY, 
-        w_p2p_bonus=W_P2P_BONUS
-    )
-
-    # Data collectors
-    all_logs = []
-    daily_summaries = []
-    step_timing_list = []
-
-    # Per-day evaluation
-    evaluation_start = time.time()
-    for day in range(1, DAYS_TO_EVALUATE + 1):
-        obs_clusters, _ = cluster_env.reset()
-        done_all = False
-        step_count = 0
-        day_logs = []
-
-        while not done_all and step_count < eval_num_steps:
-            step_start_time = time.time()
-            step_count += 1
-
-            # Get high-level actions
-            inter_cluster_obs_local_list = [coordinator.get_cluster_state(se, step_count) for se in cluster_env.cluster_envs]
-            inter_cluster_obs_local = np.array(inter_cluster_obs_local_list)
-            
-            # Create the global state for the high-level agent
-            inter_cluster_obs_global = inter_cluster_obs_local.flatten()
-
-            with torch.no_grad():
-                # Call select_action with both local and global states
-                high_level_action, _ = inter_agent.select_action(
-                    inter_cluster_obs_local,
-                    inter_cluster_obs_global
-                )
-
-            # Build transfers
-            current_reports = {i: {'export_capacity': cluster_env.get_export_capacity(i), 'import_capacity': cluster_env.get_import_capacity(i)} for i in range(n_clusters)}
-            exports, imports = coordinator.build_transfers(high_level_action, current_reports)
-
-            # Get low-level actions
-            batch_global_obs = obs_clusters.reshape(n_clusters, -1)
-            with torch.no_grad():
-                low_level_actions_list = []
-                for c_idx in range(n_clusters):
-                    agent = low_agents[c_idx]
-                    local_obs_cluster = obs_clusters[c_idx]
-                    global_obs_cluster = batch_global_obs[c_idx]
-                    actions, _ = agent.select_action(local_obs_cluster, global_obs_cluster)
-                    low_level_actions_list.append(actions)
-                low_level_actions = np.stack(low_level_actions_list)
-
-            # Step the environment
-            next_obs, rewards, done_all, step_info = cluster_env.step(
-                low_level_actions, exports=exports, imports=imports
-            )
-
-            # Advance the state
-            obs_clusters = next_obs
-            
-            # Timing and console printout
-            step_duration = time.time() - step_start_time
-            print(f"[Day {day}, Step {step_count}] Step time: {step_duration:.6f} seconds")
-            step_timing_list.append({"day": day, "step": step_count, "step_time_s": step_duration})
-
-            # Consolidated Logging
-            infos = step_info.get("cluster_infos")
-            for c_idx, subenv in enumerate(cluster_env.cluster_envs):
-                grid_price_now = subenv.get_grid_price(step_count - 1)
-                peer_price_now = step_info.get("peer_price_global")
-                if peer_price_now is None:
-                    demands_step = subenv.demands_day[step_count-1]
-                    solars_step = subenv.solars_day[step_count-1]
-                    surplus = np.maximum(solars_step - demands_step, 0.0).sum()
-                    shortfall = np.maximum(demands_step - solars_step, 0.0).sum()
-                    peer_price_now = subenv.get_peer_price(step_count -1, surplus, shortfall)
-
-                for i, hid in enumerate(subenv.house_ids):
-                    is_battery_house = hid in subenv.batteries
-                    charge = infos["charge_amount"][c_idx][i]
-                    discharge = infos["discharge_amount"][c_idx][i]
-                    day_logs.append({
-                        "day": day, 
-                        "step": step_count - 1, 
-                        "house": hid, 
-                        "cluster": c_idx,
-                        "grid_import_no_p2p": infos["grid_import_no_p2p"][c_idx][i],
-                        "grid_import_with_p2p": infos["grid_import_with_p2p"][c_idx][i],
-                        "grid_export": infos["grid_export"][c_idx][i],
-                        "p2p_buy": infos["p2p_buy"][c_idx][i], 
-                        "p2p_sell": infos["p2p_sell"][c_idx][i],
-                        "actual_cost": infos["costs"][c_idx][i],
-                        "baseline_cost": infos["grid_import_no_p2p"][c_idx][i] * grid_price_now,
-                        "total_demand": subenv.demands_day[step_count-1, i],
-                        "total_solar": subenv.solars_day[step_count-1, i],
-                        "grid_price": grid_price_now, 
-                        "peer_price": peer_price_now,
-                        "soc": (subenv.battery_soc[i] / subenv.battery_max_capacity[i]) if is_battery_house and subenv.battery_max_capacity[i] > 0 else np.nan,
-                        "degradation_cost": (charge + discharge) * subenv.battery_degradation_cost[i] if is_battery_house else 0.0,
-                        "reward": infos["agent_rewards"][c_idx][i],
-                    })
-
-            step_duration = time.time() - step_start_time
-
-        # End of day: aggregate & summarize
-        df_day = pd.DataFrame(day_logs)
-        if df_day.empty:
-            continue
-        all_logs.extend(day_logs)
-
-        # Consolidated Daily Summary Calculation
-        # Correctly count solar houses from the daily data
-        num_solar_houses = df_day[df_day['total_solar'] > 0]['house'].nunique()
-
-        if num_solar_houses > 0:
-            # Get the total number of agents for scaling the threshold
-            num_agents_in_day = df_day['house'].nunique()
-            
-            # Calculate aggregate solar generation per step
-            agg_solar_per_step = df_day.groupby("step")["total_solar"].sum()
-            
-            # Find steps where aggregate solar exceeds the scaled threshold
-            sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents_in_day)
-            sunny_steps = sunny_steps_mask[sunny_steps_mask].index
-
-            # The rest of the calculation remains the same
-            trade_df = df_day[df_day["step"].isin(sunny_steps)]
-
-        grouped_house = df_day.groupby("house").sum(numeric_only=True)
-        grouped_step = df_day.groupby("step").sum(numeric_only=True)
-
-        total_demand = grouped_step["total_demand"].sum()
-        total_solar = grouped_step["total_solar"].sum()
-        total_p2p_buy = df_day['p2p_buy'].sum()
-        total_p2p_sell = df_day['p2p_sell'].sum()
-        total_actual_grid_import = df_day['grid_import_with_p2p'].sum()
-
-        baseline_cost_per_house = grouped_house["baseline_cost"]
-        actual_cost_per_house = grouped_house["actual_cost"]
-        cost_savings_per_house = baseline_cost_per_house - actual_cost_per_house
-        day_total_cost_savings = cost_savings_per_house.sum()
-        
-        if baseline_cost_per_house.sum() > 0:
-            overall_cost_savings_pct = day_total_cost_savings / baseline_cost_per_house.sum()
-        else:
-            overall_cost_savings_pct = 0.0
-
-        baseline_import_per_house = grouped_house["grid_import_no_p2p"]
-        actual_import_per_house = grouped_house["grid_import_with_p2p"]
-        import_reduction_per_house = baseline_import_per_house - actual_import_per_house
-        day_total_import_reduction = import_reduction_per_house.sum()
-        
-        if baseline_import_per_house.sum() > 0:
-            overall_import_reduction_pct = day_total_import_reduction / baseline_import_per_house.sum()
-        else:
-            overall_import_reduction_pct = 0.0
-
-        fairness_cost_savings = compute_jains_fairness(cost_savings_per_house.values)
-        fairness_import_reduction = compute_jains_fairness(import_reduction_per_house.values)
-        fairness_rewards = compute_jains_fairness(grouped_house["reward"].values)
-        fairness_p2p_buy = compute_jains_fairness(grouped_house["p2p_buy"].values)
-        fairness_p2p_sell = compute_jains_fairness(grouped_house["p2p_sell"].values)
-        fairness_p2p_total = compute_jains_fairness((grouped_house["p2p_buy"] + grouped_house["p2p_sell"]).values)
-
-        daily_summaries.append({
-            "day": day,
-            "day_total_demand": total_demand,
-            "day_total_solar": total_solar,
-            "day_p2p_buy": total_p2p_buy,
-            "day_p2p_sell": total_p2p_sell,
-            "cost_savings_abs": day_total_cost_savings,
-            "cost_savings_pct": overall_cost_savings_pct,
-            "fairness_cost_savings": fairness_cost_savings,
-            "grid_reduction_abs": day_total_import_reduction,
-            "grid_reduction_pct": overall_import_reduction_pct,
-            "fairness_grid_reduction": fairness_import_reduction,
-            "fairness_reward": fairness_rewards,
-            "fairness_p2p_buy": fairness_p2p_buy,
-            "fairness_p2p_sell": fairness_p2p_sell,
-            "fairness_p2p_total": fairness_p2p_total,
-        })
-
-    # Final Processing and Saving
-    evaluation_end = time.time()
-    total_eval_time = evaluation_end - evaluation_start
-    print(f"\nEvaluation loop finished. Total time: {total_eval_time:.2f} seconds.")
-
-    all_days_df = pd.DataFrame(all_logs)
-    if not all_days_df.empty:
-        # Save step-level logs
-        combined_csv_path = os.path.join(logs_dir, "step_logs_all_days.csv")
-        all_days_df.to_csv(combined_csv_path, index=False)
-        print(f"Saved combined step-level logs to: {combined_csv_path}")
-
-        # Save timing logs
-        step_timing_df = pd.DataFrame(step_timing_list)
-        timing_csv_path = os.path.join(logs_dir, "step_timing_log.csv")
-        step_timing_df.to_csv(timing_csv_path, index=False)
-        print(f"Saved step timing logs to: {timing_csv_path}")
-
-        # Save house-level summary
-        house_level_df = all_days_df.groupby("house").agg({
-            "baseline_cost": "sum",
-            "actual_cost": "sum",
-            "grid_import_no_p2p": "sum",
-            "grid_import_with_p2p": "sum",
-            "degradation_cost": "sum"
-        })
-        house_level_df["cost_savings"] = house_level_df["baseline_cost"] - house_level_df["actual_cost"]
-        house_level_df["import_reduction"] = house_level_df["grid_import_no_p2p"] - house_level_df["grid_import_with_p2p"]
-        house_summary_csv = os.path.join(logs_dir, "summary_per_house.csv")
-        house_level_df.to_csv(house_summary_csv)
-        print(f"Saved final summary per house to: {house_summary_csv}")
-
-        # Calculate Final Summary Metrics
-        daily_summary_df = pd.DataFrame(daily_summaries)
-
-        fairness_grid_all = compute_jains_fairness(house_level_df["import_reduction"].values)
-        fairness_cost_all = compute_jains_fairness(house_level_df["cost_savings"].values)
-
-        total_cost_savings_all = daily_summary_df["cost_savings_abs"].sum()
-        total_baseline_cost_all = all_days_df.groupby('day')['baseline_cost'].sum().sum()
-        pct_cost_savings_all = total_cost_savings_all / total_baseline_cost_all if total_baseline_cost_all > 0 else 0.0
-
-        total_grid_reduction_all = daily_summary_df["grid_reduction_abs"].sum()
-        total_baseline_import_all = all_days_df.groupby('day')['grid_import_no_p2p'].sum().sum()
-        pct_grid_reduction_all = total_grid_reduction_all / total_baseline_import_all if total_baseline_import_all > 0 else 0.0
-
-        total_degradation_cost_all = all_days_df["degradation_cost"].sum()
-
-        # Calculate Alternative Performance Metrics
-        agg_solar_per_step = all_days_df.groupby(['day', 'step'])['total_solar'].sum()
-        num_agents_total = len(all_days_df['house'].unique())
-        sunny_steps_mask = agg_solar_per_step > (SOLAR_THRESHOLD * num_agents_total)
-        sunny_df = all_days_df[all_days_df.set_index(['day', 'step']).index.isin(sunny_steps_mask[sunny_steps_mask].index)]
-
-        baseline_import_sunny = sunny_df['grid_import_no_p2p'].sum()
-        actual_import_sunny = sunny_df['grid_import_with_p2p'].sum()
-        grid_reduction_sunny_pct = (baseline_import_sunny - actual_import_sunny) / baseline_import_sunny if baseline_import_sunny > 0 else 0.0
-
-        total_p2p_buy = all_days_df['p2p_buy'].sum()
-        total_actual_grid_import = all_days_df['grid_import_with_p2p'].sum()
-        community_sourcing_rate_pct = total_p2p_buy / (total_p2p_buy + total_actual_grid_import) if (total_p2p_buy + total_actual_grid_import) > 0 else 0.0
-
-        total_p2p_sell = all_days_df['p2p_sell'].sum()
-        total_grid_export = all_days_df['grid_export'].sum()
-        solar_sharing_efficiency_pct = total_p2p_sell / (total_p2p_sell + total_grid_export) if (total_p2p_sell + total_grid_export) > 0 else 0.0
-
-        # Calculate cost savings in sunny hours
-        baseline_cost_sunny = sunny_df['baseline_cost'].sum()
-        actual_cost_sunny = sunny_df['actual_cost'].sum()
-        cost_savings_sunny_pct = (baseline_cost_sunny - actual_cost_sunny) / baseline_cost_sunny if baseline_cost_sunny > 0 else 0.0
-
-        # Create and Save Final Summary CSV
-        final_row = {
-            "day": "ALL_DAYS_SUMMARY", 
-            "cost_savings_abs": total_cost_savings_all, 
-            "cost_savings_pct": pct_cost_savings_all,
-            "grid_reduction_abs": total_grid_reduction_all, 
-            "grid_reduction_pct": pct_grid_reduction_all,
-            "fairness_cost_savings": fairness_cost_all, 
-            "fairness_grid_reduction": fairness_grid_all,
-            "total_degradation_cost": total_degradation_cost_all,
-            "grid_reduction_sunny_hours_pct": grid_reduction_sunny_pct,
-            "community_sourcing_rate_pct": community_sourcing_rate_pct,
-            "solar_sharing_efficiency_pct": solar_sharing_efficiency_pct,
-        }
-        final_row_df = pd.DataFrame([final_row])
-
-        # Ensure daily summary has columns before concatenating
-        if not daily_summary_df.empty:
-            daily_summary_df = pd.concat([daily_summary_df, final_row_df], ignore_index=True)
-
-        summary_csv = os.path.join(logs_dir, "summary_per_day.csv")
-        daily_summary_df.to_csv(summary_csv, index=False)
-        print(f"Saved day-level summary with final multi-day row to: {summary_csv}")
-
-        # Final Printout
-        print("\n================== EVALUATION SUMMARY ==================")
-        print(f"Evaluation finished for {DAYS_TO_EVALUATE} days.\n")
-        print("--- Standard Metrics (24-Hour Average) ---")
-        print(f"Total grid reduction: {total_grid_reduction_all:.2f} kWh ({pct_grid_reduction_all:.2%})")
-        print(f"Total cost savings: ${total_cost_savings_all:.2f} ({pct_cost_savings_all:.2%})")
-        print(f"Jain's fairness on grid reduction: {fairness_grid_all:.3f}")
-        print(f"Jain's fairness on cost savings:   {fairness_cost_all:.3f}\n")
-        print("--- Alternative Metrics (Highlighting Peak Performance) ---")
-        print(f"Grid reduction during solar hours: {grid_reduction_sunny_pct:.2%}")
-        print(f"Cost savings during solar hours:   {cost_savings_sunny_pct:.2%}") 
-        print(f"Community sourcing rate:           {community_sourcing_rate_pct:.2%}")
-        print(f"Solar sharing efficiency:          {solar_sharing_efficiency_pct:.2%}")
-        print("=========================================================")
-
-        # Generate Plots
-        # Create a clean version of the daily summary for plotting, excluding the final summary row
-        plot_daily_df = daily_summary_df[daily_summary_df["day"] != "ALL_DAYS_SUMMARY"].copy()
-        plot_daily_df["day"] = plot_daily_df["day"].astype(int)
-
-        # Plot 1: Daily Cost Savings Percentage
-        plt.figure(figsize=(12, 6))
-        plt.bar(plot_daily_df["day"], plot_daily_df["cost_savings_pct"] * 100, color='skyblue')
-        plt.xlabel("Day")
-        plt.ylabel("Cost Savings (%)")
-        plt.title("Daily Community Cost Savings Percentage")
-        plt.xticks(plot_daily_df["day"])
-        plt.grid(axis='y', linestyle='--', alpha=0.7)
-        plt.savefig(os.path.join(plots_dir, "daily_cost_savings_percentage.png"))
-        plt.close()
-
-        # Plot 2: Daily Total Demand vs. Solar
-        plt.figure(figsize=(12, 6))
-        bar_width = 0.4
-        days = plot_daily_df["day"]
-        plt.bar(days - bar_width/2, plot_daily_df["day_total_demand"], width=bar_width, label="Total Demand", color='coral')
-        plt.bar(days + bar_width/2, plot_daily_df["day_total_solar"],  width=bar_width, label="Total Solar Generation", color='gold')
-        plt.xlabel("Day")
-        plt.ylabel("Energy (kWh)")
-        plt.title("Total Community Demand vs. Solar Generation Per Day")
-        plt.xticks(days)
-        plt.legend()
-        plt.grid(axis='y', linestyle='--', alpha=0.7)
-        plt.savefig(os.path.join(plots_dir, "daily_demand_vs_solar.png"))
-        plt.close()
-
-        # Plot 3: Combined Time Series of Energy Flows
-        # Aggregate data by global step across all days
-        step_group = all_days_df.groupby(["day", "step"]).sum(numeric_only=True).reset_index()
-        step_group["global_step"] = (step_group["day"] - 1) * eval_num_steps + step_group["step"]
-        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 12), sharex=True)
-        
-        # Subplot 1: Grid Import vs P2P Buy
-        ax1.plot(step_group["global_step"], step_group["grid_import_with_p2p"], label="Grid Import (with P2P)", color='r')
-        ax1.plot(step_group["global_step"], step_group["p2p_buy"], label="P2P Buy", color='g')
-        ax1.set_ylabel("Energy (kWh)")
-        ax1.set_title("Community Energy Consumption: Grid Import vs. P2P Buy")
-        ax1.legend()
-        ax1.grid(True, linestyle='--', alpha=0.6)
-
-        # Subplot 2: Grid Export vs P2P Sell
-        ax2.plot(step_group["global_step"], step_group["grid_export"], label="Grid Export", color='orange')
-        ax2.plot(step_group["global_step"], step_group["p2p_sell"], label="P2P Sell", color='b')
-        ax2.set_xlabel("Global Timestep")
-        ax2.set_ylabel("Energy (kWh)")
-        ax2.set_title("Community Energy Generation: Grid Export vs. P2P Sell")
-        ax2.legend()
-        ax2.grid(True, linestyle='--', alpha=0.6)
-        
-        plt.tight_layout()
-        plt.savefig(os.path.join(plots_dir, "combined_energy_flows_timeseries.png"))
-        plt.close()
-
-        # Plot 4: Stacked Bar of Daily Energy Sources
-        # Shows how the community's baseline grid import is met by actual grid import vs. P2P trading
-        daily_agg = all_days_df.groupby("day").sum(numeric_only=True)
-        
-        plt.figure(figsize=(12, 7))
-        plt.bar(daily_agg.index, daily_agg["grid_import_with_p2p"], label="Grid Import (with P2P)", color='crimson')
-        plt.bar(daily_agg.index, daily_agg["p2p_buy"], bottom=daily_agg["grid_import_with_p2p"], label="P2P Buy", color='limegreen')
-        plt.plot(daily_agg.index, daily_agg["grid_import_no_p2p"], label="Baseline Grid Import (No P2P)", color='blue', linestyle='--', marker='o')
-        
-        plt.xlabel("Day")
-        plt.ylabel("Energy (kWh)")
-        plt.title("Daily Energy Procurement: Baseline vs. P2P+Grid")
-        plt.xticks(daily_agg.index)
-        plt.legend()
-        plt.grid(axis='y', linestyle='--', alpha=0.7)
-        plt.savefig(os.path.join(plots_dir, "daily_energy_procurement_stacked.png"))
-        plt.close()
-
-        # Plot 5: Fairness Metrics Over Time
-        plt.figure(figsize=(12, 6))
-        plt.plot(plot_daily_df["day"], plot_daily_df["fairness_cost_savings"], label="Cost Savings Fairness", marker='o')
-        plt.plot(plot_daily_df["day"], plot_daily_df["fairness_grid_reduction"], label="Grid Reduction Fairness", marker='s')
-        plt.plot(plot_daily_df["day"], plot_daily_df["fairness_reward"], label="Reward Fairness", marker='^')
-        plt.xlabel("Day")
-        plt.ylabel("Jain's Fairness Index")
-        plt.title("Daily Fairness Metrics")
-        plt.xticks(plot_daily_df["day"])
-        plt.ylim(0, 1.05)
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.7)
-        plt.savefig(os.path.join(plots_dir, "daily_fairness_metrics.png"))
-        plt.close()
-
-        # Plot 6: Per-House Savings and Reductions
-        # Uses the house_level_df which summarizes stats over all evaluated days
-        fig, ax1 = plt.subplots(figsize=(15, 7))
-        
-        house_ids_str = house_level_df.index.astype(str)
-        bar_width = 0.4
-        index = np.arange(len(house_ids_str))
-
-        # Bar chart for cost savings
-        color1 = 'tab:green'
-        ax1.set_xlabel('House ID')
-        ax1.set_ylabel('Total Cost Savings ($)', color=color1)
-        ax1.bar(index - bar_width/2, house_level_df["cost_savings"], bar_width, label='Cost Savings', color=color1)
-        ax1.tick_params(axis='y', labelcolor=color1)
-        ax1.set_xticks(index)
-        ax1.set_xticklabels(house_ids_str, rotation=45, ha="right")
-        
-        # Instantiate a second y-axis for grid import reduction
-        ax2 = ax1.twinx()
-        color2 = 'tab:blue'
-        ax2.set_ylabel('Total Grid Import Reduction (kWh)', color=color2)
-        ax2.bar(index + bar_width/2, house_level_df["import_reduction"], bar_width, label='Import Reduction', color=color2)
-        ax2.tick_params(axis='y', labelcolor=color2)
-
-        plt.title(f'Total Cost Savings & Grid Import Reduction Per House (over {DAYS_TO_EVALUATE} days)')
-
-        fig.tight_layout()
-        plt.savefig(os.path.join(plots_dir, "per_house_summary.png"))
-        plt.close()
-        
-        # Plot 7: Price Dynamics for a Single Day
-        # Visualize the prices the agents see on the first day of evaluation
-        day1_prices = all_days_df[all_days_df['day'] == 1][['step', 'grid_price', 'peer_price']].drop_duplicates()
-        plt.figure(figsize=(12, 6))
-        plt.plot(day1_prices['step'], day1_prices['grid_price'], label='Grid Price', color='darkorange')
-        plt.plot(day1_prices['step'], day1_prices['peer_price'], label='P2P Price', color='teal')
-        plt.xlabel("Timestep of Day")
-        plt.ylabel("Price ($/kWh)")
-        plt.title("Price Dynamics on Day 1")
-        plt.legend()
-        plt.grid(True, linestyle='--', alpha=0.6)
-        plt.savefig(os.path.join(plots_dir, "price_dynamics_day1.png"))
-        plt.close()
-        
-        # Plot 8: Battery State of Charge (SoC) for a Sample of Houses
-        day1_df = all_days_df[all_days_df['day'] == 1]
-        battery_houses = day1_df.dropna(subset=['soc'])['house'].unique()
-        
-        if len(battery_houses) > 0:
-            sample_houses = battery_houses[:min(4, len(battery_houses))]  # Plot up to 4 houses
-            plt.figure(figsize=(12, 6))
-            for house in sample_houses:
-                house_df = day1_df[day1_df['house'] == house]
-                plt.plot(house_df['step'], house_df['soc'] * 100, label=f'House {house}')
-            
-            plt.xlabel("Timestep of Day")
-            plt.ylabel("State of Charge (%)")
-            plt.title("Battery SoC on Day 1 for Sample Houses")
-            plt.legend()
-            plt.grid(True, linestyle='--', alpha=0.6)
-            plt.savefig(os.path.join(plots_dir, "soc_dynamics_day1.png"))
-            plt.close()
-
-        print("All plots have been generated and saved. Evaluation complete.")
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/HC_MAPPO/HC_MAPPO_train.py

@@ -1,579 +0,0 @@
-import os
-import sys
-import time
-from datetime import datetime, timedelta
-import re
-
-import numpy as np
-import torch
-import pandas as pd
-import matplotlib.pyplot as plt
-
-# Allow imports from project root
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from cluster import InterClusterCoordinator, InterClusterLedger
-from Environment.cluster_env_wrapper import make_vec_env 
-from mappo.trainer.mappo import MAPPO
-
-
-def recursive_sum(item):
-    total = 0
-    # Check if the item is a list, array, or other iterable, but not a string
-    if hasattr(item, '__iter__') and not isinstance(item, str):
-        for sub_item in item:
-            total += recursive_sum(sub_item)
-    # If it's a single number, just add it
-    elif np.isreal(item):
-        total += item
-    # Ignore any non-numeric, non-iterable items
-    return total
-
-
-def main():
-    overall_start_time = time.time()
-    
-    # Training Configuration Parameters
-    STATE_TO_RUN = "oklahoma"  # or "colorado", "oklahoma"
-    DATA_PATH = "data/training/1000houses_152days_TRAIN.csv"
-    
-    # Dynamically extract the number of agents from the file path
-    match = re.search(r'(\d+)houses', DATA_PATH)
-    if not match:
-        raise ValueError("Could not extract the number of houses from DATA_PATH.")
-    NUMBER_OF_AGENTS = int(match.group(1))
-
-    CLUSTER_SIZE = 10
-    NUM_EPISODES = 10000
-    BATCH_SIZE = 256
-    CHECKPOINT_INTERVAL = 100000  # Reduced for more frequent saving during testing
-    WINDOW_SIZE = 80
-    MAX_TRANSFER_KWH = 100000
-
-    LR = 2e-4
-    GAMMA = 0.95
-    LAMBDA = 0.95
-    CLIP_EPS = 0.2
-    K_EPOCHS = 4
-    
-    JOINT_TRAINING_START_EPISODE = 2000
-    FREEZE_HIGH_FOR_EPISODES = 20
-    FREEZE_LOW_FOR_EPISODES = 10
-
-    # Build run directories
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    run_name = f"hierarchical_{STATE_TO_RUN}_{NUMBER_OF_AGENTS}agents_" \
-               f"{CLUSTER_SIZE}size_{NUM_EPISODES}eps_{timestamp}"
-    root_dir = os.path.join("FINALE_FINALE_FINALE", run_name)  # New folder for new runs
-    models_dir = os.path.join(root_dir, "models")
-    logs_dir = os.path.join(root_dir, "logs")
-    plots_dir = os.path.join(root_dir, "plots")
-
-    for d in (models_dir, logs_dir, plots_dir):
-        os.makedirs(d, exist_ok=True)
-    print(f"Logging to: {root_dir}")
-
-    # Environment & Agent Initialization
-    
-    # Instantiate the environment using vectorized environment factory function
-    # This single call replaces the manual creation of base_env and ClusterEnvWrapper
-    cluster_env = make_vec_env(
-        data_path=DATA_PATH,
-        time_freq="15T",
-        cluster_size=CLUSTER_SIZE,
-        state=STATE_TO_RUN
-    )
-
-    # Get environment parameters from the vectorized environment object
-    n_clusters = cluster_env.num_envs
-    sample_subenv = cluster_env.cluster_envs[0]  # Access a sample sub-env
-    n_agents_per_cluster = sample_subenv.num_agents
-
-    local_dim = sample_subenv.observation_space.shape[-1]
-    global_dim = n_agents_per_cluster * local_dim
-    # Access the action dim from the first part of the Tuple action space
-    act_dim = sample_subenv.action_space[0].shape[-1] 
-    # The total number of transitions collected each episode is (steps_per_day * num_clusters)
-    total_buffer_size = sample_subenv.num_steps * n_clusters 
-    print(f"Low-level agent buffer size set to: {total_buffer_size}")
-
-    print(f"Created {n_clusters} clusters.")
-    print(f"Shared low-level agent: {n_agents_per_cluster} agents per cluster, "
-          f"obs_dim={local_dim}, global_dim={global_dim}, act_dim={act_dim}")
-
-    print(f"Creating {n_clusters} independent low-level MAPPO agents...")
-    low_agents = []
-    for i in range(n_clusters):
-        # Each agent's buffer only needs to be as long as one episode day
-        agent_buffer_size = sample_subenv.num_steps 
-
-        agent = MAPPO(
-            n_agents=n_agents_per_cluster,
-            local_dim=local_dim,
-            global_dim=global_dim,
-            act_dim=act_dim,
-            lr=LR,
-            gamma=GAMMA,
-            lam=LAMBDA,
-            clip_eps=CLIP_EPS,
-            k_epochs=K_EPOCHS,
-            batch_size=BATCH_SIZE,
-            episode_len=agent_buffer_size 
-        )
-        low_agents.append(agent)
-
-    # Define dimensions for the high-level MAPPO agent
-    OBS_DIM_HI_LOCAL = 7   # Each cluster has 7 features for its local state
-    act_dim_inter = 2      # Export/Import preference for each cluster
-    
-    # The global state for the high-level agent is the concatenation
-    # of all high-level local states
-    OBS_DIM_HI_GLOBAL = n_clusters * OBS_DIM_HI_LOCAL
-
-    print(f"Inter-cluster agent (MAPPO): n_agents={n_clusters}, "
-          f"local_dim={OBS_DIM_HI_LOCAL}, global_dim={OBS_DIM_HI_GLOBAL}, act_dim={act_dim_inter}")
-
-    # Instantiate MAPPO for the inter-cluster agent
-    inter_agent = MAPPO(
-        n_agents=n_clusters,
-        local_dim=OBS_DIM_HI_LOCAL,
-        global_dim=OBS_DIM_HI_GLOBAL,
-        act_dim=act_dim_inter,
-        lr=LR,
-        gamma=GAMMA,
-        lam=LAMBDA,
-        clip_eps=CLIP_EPS,
-        k_epochs=K_EPOCHS,
-        batch_size=BATCH_SIZE,
-        episode_len=sample_subenv.num_steps
-    )
-
-    ledger = InterClusterLedger()
-    coordinator = InterClusterCoordinator(
-        cluster_env,
-        inter_agent,
-        ledger,
-        max_transfer_kwh=MAX_TRANSFER_KWH
-    )
-
-    # Training loop
-    total_steps = 0
-    episode_log_data = []
-    performance_metrics_log = []
-    intra_log = {}
-    inter_log = {}
-    total_log = {}
-    cost_log = {}
-
-    for ep in range(1, NUM_EPISODES + 1):
-        step_count = 0
-        start_time = time.time()
-        ep_total_inter_cluster_reward = 0.0
-        day_logs = []
-        
-        obs_clusters, _ = cluster_env.reset()
-
-        if ep > 1:
-            # For vectorized envs, call is the right way to invoke a method on all sub-envs
-            # This returns a list of dictionaries, one from each cluster env
-            all_cluster_metrics = cluster_env.call('get_episode_metrics')
-
-            # Aggregate the metrics from all clusters into a single system-wide summary
-            system_metrics = {
-                "grid_reduction_entire_day": sum(m["grid_reduction_entire_day"] for m in all_cluster_metrics),
-                "grid_reduction_peak_hours": sum(m["grid_reduction_peak_hours"] for m in all_cluster_metrics),
-                "total_cost_savings": sum(m["total_cost_savings"] for m in all_cluster_metrics),
-                "battery_degradation_cost_total": sum(m["battery_degradation_cost_total"] for m in all_cluster_metrics),
-                # For fairness, we average the fairness index across clusters
-                "fairness_on_cost_savings": np.mean([m["fairness_on_cost_savings"] for m in all_cluster_metrics]),
-                "Episode": ep - 1  # Associate with the episode that just finished
-            }
-
-            # Append the aggregated dictionary to our log
-            performance_metrics_log.append(system_metrics)
-        
-        # Use a single 'done' flag for the episode
-        done_all = False
-        
-        # Initialize rewards and costs
-        cluster_rewards = np.zeros((n_clusters, n_agents_per_cluster), dtype=np.float32)
-        total_cost = 0.0
-        total_grid_import = 0.0
-        
-        # Determine training phase
-        is_phase_1 = ep < JOINT_TRAINING_START_EPISODE
-
-        if ep == 1: 
-            print(f"\n--- Starting Phase 1: Training Low-Level Agent Only (up to ep {JOINT_TRAINING_START_EPISODE-1}) ---")
-        if ep == JOINT_TRAINING_START_EPISODE: 
-            print(f"\n--- Starting Phase 2: Joint Hierarchical Training (from ep {JOINT_TRAINING_START_EPISODE}) ---")
-        
-        # The main loop continues as long as the episode is not done
-        while not done_all:
-            total_steps += 1
-            step_count += 1
-            
-            # Action Selection (Low-Level)
-            batch_global_obs = obs_clusters.reshape(n_clusters, -1)
-            low_level_actions_list = []
-            low_level_logps_list = []
-            
-            for c_idx in range(n_clusters):
-                agent = low_agents[c_idx]
-                local_obs_cluster = obs_clusters[c_idx]
-                global_obs_cluster = batch_global_obs[c_idx]
-                actions, logps = agent.select_action(local_obs_cluster, global_obs_cluster)
-                low_level_actions_list.append(actions)
-                low_level_logps_list.append(logps)
-
-            low_level_actions = np.stack(low_level_actions_list)
-            low_level_logps = np.stack(low_level_logps_list)
-
-            # Action Selection & Transfers (High-Level, Phase 2 only)
-            if is_phase_1:
-                exports, imports = None, None
-            else:
-                inter_cluster_obs_local_list = [coordinator.get_cluster_state(se, step_count) for se in cluster_env.cluster_envs]
-                inter_cluster_obs_local = np.array(inter_cluster_obs_local_list)
-                
-                # Create the global state for the high-level agent
-                inter_cluster_obs_global = inter_cluster_obs_local.flatten()
-
-                # Call select_action with local and global states
-                high_level_action, high_level_logp = inter_agent.select_action(
-                    inter_cluster_obs_local,
-                    inter_cluster_obs_global
-                )
-
-                current_reports = {i: {'export_capacity': cluster_env.get_export_capacity(i), 'import_capacity': cluster_env.get_import_capacity(i)} for i in range(n_clusters)}
-                exports, imports = coordinator.build_transfers(high_level_action, current_reports)
-
-            # Environment Step
-            next_obs_clusters, rewards, done_all, step_info = cluster_env.step(
-                low_level_actions, exports=exports, imports=imports
-            )
-            cluster_infos = step_info.get("cluster_infos")
-            day_logs.append({
-                "costs": cluster_infos["costs"],
-                "grid_import_no_p2p": cluster_infos["grid_import_no_p2p"],
-                "charge_amount": cluster_infos.get("charge_amount"),
-                "discharge_amount": cluster_infos.get("discharge_amount")
-            })
-
-            # Reward Calculation and Data Storage
-            per_agent_rewards = np.stack(cluster_infos['agent_rewards'])
-            rewards_for_buffer = per_agent_rewards
-
-            if not is_phase_1:
-                transfers_for_logging = (exports, imports)
-                high_level_rewards_per_cluster = coordinator.compute_inter_cluster_reward(
-                    all_cluster_infos=cluster_infos,
-                    actual_transfers=transfers_for_logging,
-                    step_count=step_count
-                )
-                ep_total_inter_cluster_reward += np.sum(high_level_rewards_per_cluster)
-
-                # Get next state for high-level agent's buffer
-                next_inter_cluster_obs_local_list = [coordinator.get_cluster_state(se, step_count + 1) for se in cluster_env.cluster_envs]
-                next_inter_cluster_obs_local = np.array(next_inter_cluster_obs_local_list)
-                
-                # Create the next global state
-                next_inter_cluster_obs_global = next_inter_cluster_obs_local.flatten()
-
-                # Store the transition in the high-level MAPPO agent's buffer
-                inter_agent.store(
-                    inter_cluster_obs_local,       # s_local
-                    inter_cluster_obs_global,      # s_global
-                    high_level_action,             # action
-                    high_level_logp,               # log_prob
-                    high_level_rewards_per_cluster,# reward
-                    [done_all] * n_clusters,       # done
-                    next_inter_cluster_obs_global  # s'_global
-                )
-
-                bonus_per_agent = np.zeros_like(per_agent_rewards)
-                for c_idx in range(n_clusters):
-                    num_agents_in_cluster = per_agent_rewards.shape[1]
-                    if num_agents_in_cluster > 0:
-                        bonus = high_level_rewards_per_cluster[c_idx] / num_agents_in_cluster
-                        bonus_per_agent[c_idx, :] = bonus
-                rewards_for_buffer = per_agent_rewards + bonus_per_agent
-
-            # Data Storage (Low-Level)
-            dones_list = step_info.get("cluster_dones")
-            for idx in range(n_clusters):
-                low_agents[idx].store(
-                    obs_clusters[idx],
-                    batch_global_obs[idx],
-                    low_level_actions[idx],
-                    low_level_logps[idx],
-                    rewards_for_buffer[idx],
-                    dones_list[idx],
-                    next_obs_clusters[idx].reshape(-1)
-                )
-
-            cluster_rewards += per_agent_rewards
-            total_cost += np.sum(cluster_infos['costs'])
-            total_grid_import += np.sum(cluster_infos['grid_import_with_p2p'])
-            obs_clusters = next_obs_clusters
-
-        # Agent Updates (End of Episode)
-        if is_phase_1:
-            for agent in low_agents:
-                agent.update()
-        else:
-            CYCLE_LENGTH = FREEZE_HIGH_FOR_EPISODES + FREEZE_LOW_FOR_EPISODES
-            phase2_episode_num = ep - JOINT_TRAINING_START_EPISODE
-            position_in_cycle = phase2_episode_num % CYCLE_LENGTH
-
-            if position_in_cycle < FREEZE_HIGH_FOR_EPISODES:
-                print(f"Updating ALL LOW-LEVEL agents (High-level is frozen).")
-                for agent in low_agents:
-                    agent.update()
-            else:
-                print(f"Updating HIGH-LEVEL agent (Low-level is frozen).")
-                inter_agent.update()
-
-        # Unified End-of-Episode Logging
-        duration = time.time() - start_time
-        num_low_level_agents = n_clusters * n_agents_per_cluster
-        get_price_fn = cluster_env.cluster_envs[0].get_grid_price
-
-        # Calculate Costs & Cost Reduction
-        # Use the recursive helper function to safely sum the broken data
-        # This is guaranteed to produce a single number for each step
-        baseline_costs_per_step = [
-            recursive_sum(entry["grid_import_no_p2p"]) * get_price_fn(i)
-            for i, entry in enumerate(day_logs)
-        ]
-        total_baseline_cost = sum(baseline_costs_per_step)
-
-        # Apply the same robust method to the actual costs
-        actual_costs_per_step = [recursive_sum(entry["costs"]) for entry in day_logs]
-        total_actual_cost = sum(actual_costs_per_step)
-        
-        cost_reduction_pct = (1 - (total_actual_cost / total_baseline_cost)) * 100 if total_baseline_cost > 0 else 0.0
-
-        # Calculate All Reward Metrics
-        # Intra-Cluster (Low-Level) Rewards
-        total_reward_intra = cluster_rewards.sum()
-        mean_reward_intra = total_reward_intra / num_low_level_agents if num_low_level_agents > 0 else 0.0
-
-        # Inter-Cluster (High-Level) Rewards
-        total_reward_inter = ep_total_inter_cluster_reward
-        mean_reward_inter = total_reward_inter / step_count if step_count > 0 else 0.0
-
-        # Total System Rewards
-        total_reward_system = total_reward_intra + total_reward_inter
-        mean_reward_system = total_reward_system / num_low_level_agents if num_low_level_agents > 0 else 0.0
-
-        # Populate Logs for Plotting (to keep generate_plots working)
-        intra_log.setdefault('total', []).append(total_reward_intra)
-        intra_log.setdefault('mean', []).append(mean_reward_intra)
-        inter_log.setdefault('total', []).append(total_reward_inter)
-        inter_log.setdefault('mean', []).append(mean_reward_inter)
-        total_log.setdefault('total', []).append(total_reward_system)
-        total_log.setdefault('mean', []).append(mean_reward_system)
-        cost_log.setdefault('total_cost', []).append(total_actual_cost)
-        cost_log.setdefault('cost_without_p2p', []).append(total_baseline_cost)
-
-        # Populate the Main Log for the Final CSV File
-        episode_log_data.append({
-            "Episode": ep,
-            "Mean_Reward_System": mean_reward_system,
-            "Mean_Reward_Intra": mean_reward_intra,
-            "Mean_Reward_Inter": mean_reward_inter,
-            "Total_Reward_System": total_reward_system,
-            "Total_Reward_Intra": total_reward_intra,
-            "Total_Reward_Inter": total_reward_inter,
-            "Cost_Reduction_Pct": cost_reduction_pct,
-            "Episode_Duration": duration,
-        })
-
-        # Print Final Episode Summary
-        print(f"Ep {ep}/{NUM_EPISODES} | "
-              f"Mean System R: {mean_reward_system:.3f} | "
-              f"Cost Red: {cost_reduction_pct:.1f}% | "
-              f"Time: {duration:.2f}s")
-
-        if ep % CHECKPOINT_INTERVAL == 0 or ep == NUM_EPISODES:
-            for c_idx, agent in enumerate(low_agents):
-                agent.save(os.path.join(models_dir, f"low_cluster{c_idx}_ep{ep}.pth"))
-            inter_agent.save(os.path.join(models_dir, f"inter_ep{ep}.pth"))
-            print(f"Saved checkpoint at episode {ep}")
-
-    print("Training completed! Aggregating final logs...")
-
-    # Capture the metrics for the very last episode
-    final_cluster_metrics = cluster_env.call('get_episode_metrics')
-    final_system_metrics = {
-        "grid_reduction_entire_day": sum(m["grid_reduction_entire_day"] for m in final_cluster_metrics),
-        "grid_reduction_peak_hours": sum(m["grid_reduction_peak_hours"] for m in final_cluster_metrics),
-        "total_cost_savings": sum(m["total_cost_savings"] for m in final_cluster_metrics),
-        "battery_degradation_cost_total": sum(m["battery_degradation_cost_total"] for m in final_cluster_metrics),
-        "fairness_on_cost_savings": np.mean([m["fairness_on_cost_savings"] for m in final_cluster_metrics]),
-        "Episode": NUM_EPISODES
-    }
-    performance_metrics_log.append(final_system_metrics)
-
-    # Create, Merge, and Save Final DataFrame
-    df_rewards_log = pd.DataFrame(episode_log_data)
-    df_perf_log = pd.DataFrame(performance_metrics_log)
-    df_final_log = pd.merge(df_rewards_log, df_perf_log, on="Episode")
-
-    log_csv_path = os.path.join(logs_dir, "training_performance_log.csv")
-
-    # Add total training time to the dataframe before saving
-    overall_end_time = time.time()
-    total_duration_seconds = overall_end_time - overall_start_time
-    total_time_row = pd.DataFrame([{"Episode": "Total_Training_Time", "Episode_Duration": total_duration_seconds}])
-    df_to_save = pd.concat([df_final_log, total_time_row], ignore_index=True)
-
-    # Reorder and select columns for the final CSV
-    columns_to_save = [
-        "Episode",
-        "Mean_Reward_System",
-        "Mean_Reward_Intra",
-        "Mean_Reward_Inter",
-        "Total_Reward_System",
-        "Total_Reward_Intra",
-        "Total_Reward_Inter",
-        "Cost_Reduction_Pct",
-        "battery_degradation_cost_total",
-        "Episode_Duration",
-        "total_cost_savings",
-        "grid_reduction_entire_day",
-        "fairness_on_cost_savings"
-    ]
-    df_to_save = df_to_save[[col for col in columns_to_save if col in df_to_save.columns]]
-    df_to_save.to_csv(log_csv_path, index=False)
-    print(f"Saved comprehensive training performance log to: {log_csv_path}")
-
-    generate_plots(
-        plots_dir=plots_dir,
-        num_episodes=NUM_EPISODES,
-        intra_log=intra_log,
-        inter_log=inter_log,
-        total_log=total_log,
-        cost_log=cost_log,
-        df_final_log=df_final_log
-    )
-    
-    overall_end_time = time.time()
-    total_duration_seconds = overall_end_time - overall_start_time
-    # Format into hours, minutes, seconds
-    total_duration_formatted = str(timedelta(seconds=int(total_duration_seconds)))
-    
-    print("\n" + "="*50)
-    print(f"Total Training Time: {total_duration_formatted} (HH:MM:SS)")
-    print("="*50)
-
-
-def generate_plots(
-        plots_dir: str,
-        num_episodes: int,
-        intra_log: dict,
-        inter_log: dict,
-        total_log: dict,
-        cost_log: list,
-        df_final_log: pd.DataFrame
-    ):
-    """
-    Generates and saves all final plots after training is complete.
-    """
-    print("Training completed! Generating plots…")
-
-    # Helper for moving average
-    def moving_avg(series, window):
-        return pd.Series(series).rolling(window=window, center=True, min_periods=1).mean().to_numpy()
-
-    ma_window = 120
-    episodes = np.arange(1, num_episodes + 1)
-
-    # Plot 1: Intra-cluster (Low-Level) Rewards
-    fig, ax = plt.subplots(figsize=(12, 7))
-    ax.plot(episodes, moving_avg(intra_log['total'], ma_window), label=f'Total Reward (MA {ma_window})', linewidth=2)
-    ax.set_xlabel("Episode")
-    ax.set_ylabel("Total Intra-Cluster Reward", color='tab:blue')
-    ax.tick_params(axis='y', labelcolor='tab:blue')
-    ax.grid(True)
-    
-    ax2 = ax.twinx()
-    ax2.plot(episodes, moving_avg(intra_log['mean'], ma_window), label=f'Mean Reward (MA {ma_window})', linewidth=2, linestyle='--', color='tab:cyan')
-    ax2.set_ylabel("Mean Intra-Cluster Reward", color='tab:cyan')
-    ax2.tick_params(axis='y', labelcolor='tab:cyan')
-    
-    fig.suptitle("Intra-Cluster (Low-Level Agent) Rewards")
-    fig.legend(loc="upper left", bbox_to_anchor=(0.1, 0.9))
-    plt.savefig(os.path.join(plots_dir, "1_intra_cluster_rewards.png"), dpi=200)
-    plt.close()
-
-    # Plot 2: Inter-cluster (High-Level) Rewards
-    fig, ax = plt.subplots(figsize=(12, 7))
-    ax.plot(episodes, moving_avg(inter_log['total'], ma_window), label=f'Total Reward (MA {ma_window})', linewidth=2, color='tab:green')
-    ax.set_xlabel("Episode")
-    ax.set_ylabel("Total Inter-Cluster Reward", color='tab:green')
-    ax.tick_params(axis='y', labelcolor='tab:green')
-    ax.grid(True)
-    
-    ax2 = ax.twinx()
-    ax2.plot(episodes, moving_avg(inter_log['mean'], ma_window), label=f'Mean Reward (MA {ma_window})', linewidth=2, linestyle='--', color='mediumseagreen')
-    ax2.set_ylabel("Mean Inter-Cluster Reward", color='mediumseagreen')
-    ax2.tick_params(axis='y', labelcolor='mediumseagreen')
-    
-    fig.suptitle("Inter-Cluster (High-Level Agent) Rewards")
-    fig.legend(loc="upper left", bbox_to_anchor=(0.1, 0.9))
-    plt.savefig(os.path.join(plots_dir, "2_inter_cluster_rewards.png"), dpi=200)
-    plt.close()
-
-    # Plot 3: Total System Rewards
-    fig, ax = plt.subplots(figsize=(12, 7))
-    ax.plot(episodes, moving_avg(total_log['total'], ma_window), label=f'Total System Reward (MA {ma_window})', linewidth=2, color='tab:red')
-    ax.set_xlabel("Episode")
-    ax.set_ylabel("Total System Reward", color='tab:red')
-    ax.tick_params(axis='y', labelcolor='tab:red')
-    ax.grid(True)
-    
-    ax2 = ax.twinx()
-    ax2.plot(episodes, moving_avg(total_log['mean'], ma_window), label=f'Mean System Reward (MA {ma_window})', linewidth=2, linestyle='--', color='salmon')
-    ax2.set_ylabel("Mean System Reward per Agent", color='salmon')
-    ax2.tick_params(axis='y', labelcolor='salmon')
-    
-    fig.suptitle("Total System Rewards (Intra + Inter)")
-    fig.legend(loc="upper left", bbox_to_anchor=(0.1, 0.9))
-    plt.savefig(os.path.join(plots_dir, "3_total_system_rewards.png"), dpi=200)
-    plt.close()
-
-    # Plot 4: Cost Reduction
-    cost_df = pd.DataFrame(cost_log)
-    cost_df['cost_reduction_pct'] = 100 * (1 - (cost_df['total_cost'] / cost_df['cost_without_p2p'])).clip(lower=-np.inf, upper=100)
-    plt.figure(figsize=(12, 7))
-    plt.plot(episodes, moving_avg(cost_df['cost_reduction_pct'], ma_window), label=f'Cost Reduction % (MA {ma_window})', color='purple', linewidth=2)
-    plt.xlabel("Episode")
-    plt.ylabel("Cost Reduction (%)")
-    plt.title("Total System-Wide Cost Reduction")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "4_cost_reduction.png"), dpi=200)
-    plt.close()
-
-    df_plot = df_final_log[pd.to_numeric(df_final_log['Episode'], errors='coerce').notna()].copy()
-    df_plot['Episode'] = pd.to_numeric(df_plot['Episode'])
-
-    # Plot 5: Battery Degradation Cost
-    plt.figure(figsize=(12, 7))
-    plt.plot(df_plot["Episode"], moving_avg(df_plot["battery_degradation_cost_total"], ma_window), 
-             label=f'Degradation Cost (MA {ma_window})', color='darkgreen', linewidth=2)
-    plt.xlabel("Episode")
-    plt.ylabel("Total Degradation Cost ($)")
-    plt.title("Total Battery Degradation Cost")
-    plt.legend()
-    plt.grid(True)
-    plt.savefig(os.path.join(plots_dir, "5_battery_degradation_cost.png"), dpi=200)
-    plt.close()
-
-    print(f"All plots have been saved to: {plots_dir}")
-
-
-if __name__ == "__main__":
-    main()

Other_algorithms/HC_MAPPO/cluster.py

@@ -1,140 +0,0 @@
-import os
-import sys
-import numpy as np
-import torch
-
-# Ensure project root is on the Python path
-# Please ensure you follow proper directory structure for running this code
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-from Environment.solar_sys_environment import SolarSys
-from Environment.cluster_env_wrapper import GlobalPriceVecEnvWrapper
-from Environment.cluster_env_wrapper import make_vec_env
-class InterClusterLedger:
-    """
-    Tracks inter-cluster debts/transfers.
-    """
-    def __init__(self):
-        self.balances = {}
-
-    def record_transfer(self, from_id: str, to_id: str, amount: float):
-        if from_id == to_id: return
-        self.balances.setdefault(from_id, {})
-        self.balances.setdefault(to_id, {})
-        self.balances[from_id][to_id] = self.balances[from_id].get(to_id, 0.0) - amount
-        self.balances[to_id][from_id] = self.balances[to_id].get(from_id, 0.0) + amount
-
-    def get_balance(self, a_id: str, b_id: str) -> float:
-        return self.balances.get(a_id, {}).get(b_id, 0.0)
-
-    def net_balances(self) -> dict:
-        return self.balances
-
-
-class InterClusterCoordinator:
-    def __init__(
-        self,
-        cluster_env, 
-        high_level_agent, 
-        ledger,
-        max_transfer_kwh: float = 1000000.0,
-        w_cost_savings: float = 2.0,
-        w_grid_penalty: float = 0.3,
-        w_p2p_bonus: float = 0.3
-    ):
-        self.cluster_env = cluster_env
-        self.agent = high_level_agent 
-        self.ledger = ledger
-        self.max_transfer_kwh = max_transfer_kwh
-        self.w_cost_savings = w_cost_savings
-        self.w_grid_penalty = w_grid_penalty
-        self.w_p2p_bonus = w_p2p_bonus
-
-    def get_cluster_state(self, env, step_count: int) -> np.ndarray:
-        """
-         array summarizing a single cluster's state by reading from its vectorized attributes.
-        """
-        solar_env = env # This is one of the vectorized SolarSys envs
-        idx = min(step_count, solar_env.num_steps - 1)
-        agg_soc = np.sum(solar_env.battery_soc)
-        agg_max_capacity = np.sum(solar_env.battery_max_capacity)
-        agg_soc_fraction = agg_soc / agg_max_capacity if agg_max_capacity > 0 else 0.0
-
-        agg_demand = np.sum(solar_env.demands_day[idx])
-        agg_solar  = np.sum(solar_env.solars_day[idx])
-
-        price = solar_env.get_grid_price(idx)
-        t_norm = idx / float(solar_env.steps_per_day)
-
-        return np.array([
-            agg_soc, agg_max_capacity, agg_soc_fraction,
-            agg_demand, agg_solar, price, t_norm
-        ], dtype=np.float32)
-
-    def build_transfers(self, agent_action_vector: np.ndarray, reports: dict) -> tuple[np.ndarray, np.ndarray]:
-        """
-        Acts as a centralized market maker based on agent actions and LIVE capacity reports.
-        """
-        n = len(self.cluster_env.clusters)
-        raw_export_prefs = agent_action_vector[:, 0]
-        raw_import_prefs = agent_action_vector[:, 1]
-
-        export_prefs = torch.softmax(torch.tensor(raw_export_prefs), dim=-1).numpy()
-        import_prefs = torch.softmax(torch.tensor(raw_import_prefs), dim=-1).numpy()
-
-        total_available_for_export = 0.0
-        potential_exports = np.zeros(n)
-        for i in range(n):
-            export_capacity = reports[i]['export_capacity']
-            pref = float(export_prefs[i])
-            potential_exports[i] = min(pref * self.max_transfer_kwh, export_capacity)
-            total_available_for_export += potential_exports[i]
-
-        total_requested_for_import = 0.0
-        potential_imports = np.zeros(n)
-        for i in range(n):
-            import_capacity = reports[i]['import_capacity']
-            pref = float(import_prefs[i])
-            potential_imports[i] = min(pref * self.max_transfer_kwh, import_capacity)
-            total_requested_for_import += potential_imports[i]
-
-        total_matched_energy = min(total_available_for_export, total_requested_for_import)
-        actual_exports = np.zeros(n)
-        actual_imports = np.zeros(n)
-
-        if total_matched_energy > 1e-6:
-            if total_available_for_export > 0:
-                actual_exports = (potential_exports / total_available_for_export) * total_matched_energy
-            if total_requested_for_import > 0:
-                actual_imports = (potential_imports / total_requested_for_import) * total_matched_energy
-
-        return actual_exports, actual_imports
-    
-    def compute_inter_cluster_reward(self, all_cluster_infos: dict, actual_transfers: tuple, step_count: int) -> np.ndarray:
-        """
-        Computes an INDIVIDUAL reward for each cluster agent to solve
-        the credit assignment problem.
-        """
-        actual_exports, actual_imports = actual_transfers
-        num_clusters = len(self.cluster_env.cluster_envs)
-        cluster_rewards = np.zeros(num_clusters, dtype=np.float32)
-
-        # Extract per-cluster cost and import data from the batched info dict
-        costs_per_cluster = [np.sum(c) for c in all_cluster_infos['costs']]
-        baseline_imports_per_cluster = [np.sum(imp) for imp in all_cluster_infos['grid_import_no_p2p']]
-        actual_imports_per_cluster = [np.sum(imp) for imp in all_cluster_infos['grid_import_with_p2p']]
-        
-        # Get the single grid price for the current step
-        grid_price = self.cluster_env.cluster_envs[0].get_grid_price(step_count)
-
-        for i in range(num_clusters):
-            baseline_cost_this_cluster = baseline_imports_per_cluster[i] * grid_price
-            actual_cost_this_cluster = costs_per_cluster[i]
-            cost_saved = baseline_cost_this_cluster - actual_cost_this_cluster
-            r_savings = self.w_cost_savings * cost_saved
-            r_grid = self.w_grid_penalty * actual_imports_per_cluster[i]
-            p2p_volume_this_cluster = actual_exports[i] + actual_imports[i]
-            r_p2p = self.w_p2p_bonus * p2p_volume_this_cluster
-            cluster_rewards[i] = r_savings + r_p2p - r_grid
-        
-        return cluster_rewards

Other_algorithms/HC_MAPPO/mappo/trainer/mappo.py

@@ -1,199 +0,0 @@
-import torch
-import torch.nn as nn
-import random
-import numpy as np
-from torch.distributions import Normal
-
-if torch.cuda.is_available():
-    device = torch.device("cuda")
-    print("Using CUDA (NVIDIA GPU)")
-else:
-    device = torch.device("cpu")
-    print("Using CPU")
-
-def set_global_seed(seed: int):
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)
-        torch.backends.cudnn.deterministic = False
-        torch.backends.cudnn.benchmark = True
-
-SEED = 42
-set_global_seed(SEED)
-
-class MLP(nn.Module):
-    def __init__(self, input_dim, hidden_dims, output_dim):
-        super().__init__()
-        layers = []
-        last_dim = input_dim
-        for h in hidden_dims:
-            layers += [nn.Linear(last_dim, h), nn.ReLU()]
-            last_dim = h
-        layers.append(nn.Linear(last_dim, output_dim))
-        self.net = nn.Sequential(*layers)
-
-    def forward(self, x):
-        return self.net(x)
-
-class Actor(nn.Module):
-    def __init__(self, obs_dim, act_dim, hidden=(64,64)):
-        super().__init__()
-        self.net = MLP(obs_dim, hidden, act_dim)
-        self.log_std = nn.Parameter(torch.zeros(act_dim))
-
-    def forward(self, x):
-        mean = self.net(x)
-        std = torch.exp(self.log_std)
-        return mean, std
-
-class Critic(nn.Module):
-    def __init__(self, state_dim, hidden=(128,128)):
-        super().__init__()
-        self.net = MLP(state_dim, hidden, 1)
-
-    def forward(self, x):
-        return self.net(x).squeeze(-1)
-
-class MAPPO:
-    def __init__(
-        self,
-        n_agents,
-        local_dim,
-        global_dim,
-        act_dim,
-        lr=3e-4,
-        gamma=0.99,
-        lam=0.95,
-        clip_eps=0.2,
-        k_epochs=10,
-        batch_size=1024,
-        episode_len=96
-    ):
-        self.n_agents = n_agents
-        self.local_dim = local_dim
-        self.global_dim = global_dim
-        self.act_dim = act_dim
-        self.gamma    = gamma
-        self.lam      = lam
-        self.clip_eps = clip_eps
-        self.k_epochs = k_epochs
-        self.batch_size = batch_size
-        self.episode_len = episode_len
-
-        self.actor  = Actor(local_dim, act_dim).to(device)
-        self.critic = Critic(global_dim).to(device)
-
-        self.opt_a = torch.optim.Adam(self.actor.parameters(), lr=lr)
-        self.opt_c = torch.optim.Adam(self.critic.parameters(), lr=lr)
-
-        print("MAPPO CUDA AMP is disabled for stability.")
-        
-        self.init_buffer()
-
-    def init_buffer(self):
-        self.ls_buf = np.zeros((self.episode_len, self.n_agents, self.local_dim), dtype=np.float16)
-        self.gs_buf = np.zeros((self.episode_len, self.global_dim), dtype=np.float16)
-        self.ac_buf = np.zeros((self.episode_len, self.n_agents, self.act_dim), dtype=np.float16)
-        self.lp_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float16)
-        self.rw_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float16)
-        self.done_buf = np.zeros((self.episode_len, self.n_agents), dtype=np.float16)
-        self.next_gs_buf = np.zeros((self.episode_len, self.global_dim), dtype=np.float16)
-        self.step_idx = 0
-
-    @torch.no_grad()
-    def select_action(self, local_obs, global_obs):
-        l = torch.from_numpy(local_obs).float().to(device)
-        mean, std = self.actor(l)
-        dist = Normal(mean, std)
-        a = dist.sample()
-        return a.cpu().numpy(), dist.log_prob(a).sum(-1).cpu().numpy()
-
-    def store(self, local_obs, global_obs, action, logp, reward, done, next_global_obs):
-        if self.step_idx < self.episode_len:
-            self.ls_buf[self.step_idx] = local_obs
-            self.gs_buf[self.step_idx] = global_obs
-            self.ac_buf[self.step_idx] = action
-            self.lp_buf[self.step_idx] = logp
-            self.rw_buf[self.step_idx] = reward
-            self.done_buf[self.step_idx] = done
-            self.next_gs_buf[self.step_idx] = next_global_obs
-            self.step_idx += 1
-
-    def compute_gae(self, T, vals):
-        N = self.n_agents
-        vals_agent = vals.unsqueeze(1).expand(-1, N).cpu().numpy()
-        next_vals_agent = np.zeros_like(vals_agent)
-        next_vals_agent[:-1] = vals_agent[1:]
-        if not self.done_buf[T-1].all():
-            with torch.no_grad():
-                v_last = self.critic(
-                    torch.from_numpy(self.next_gs_buf[T-1]).float().to(device)
-                ).cpu().item()
-                next_vals_agent[T-1, :] = v_last
-        masks = 1.0 - self.done_buf[:T]
-        rewards = self.rw_buf[:T]
-        adv = rewards + self.gamma * next_vals_agent * masks - vals_agent
-        ret = adv + vals_agent
-        adv_flat = torch.from_numpy(adv.flatten()).to(device)
-        ret_flat = torch.from_numpy(ret.flatten()).to(device)
-        return adv_flat, ret_flat
-
-    def update(self):
-        T = self.step_idx
-        if T == 0: return
-
-        gs_tensor = torch.from_numpy(self.gs_buf[:T]).float().to(device)
-        ls_tensor = torch.from_numpy(self.ls_buf[:T]).float().to(device).view(T * self.n_agents, -1)
-        ac_tensor = torch.from_numpy(self.ac_buf[:T]).float().to(device).view(T * self.n_agents, -1)
-        lp_tensor = torch.from_numpy(self.lp_buf[:T]).float().to(device).view(-1)
-        
-        with torch.no_grad():
-            vals = self.critic(gs_tensor)
-
-        adv_flat, ret_flat = self.compute_gae(T, vals)
-        adv_flat = (adv_flat - adv_flat.mean()) / (adv_flat.std() + 1e-8)
-
-        gs_for_batch = gs_tensor.unsqueeze(1).expand(-1, self.n_agents, -1).reshape(T * self.n_agents, self.global_dim)
-
-        dataset = torch.utils.data.TensorDataset(ls_tensor, gs_for_batch, ac_tensor, lp_tensor, adv_flat, ret_flat)
-        gen = torch.Generator()
-        gen.manual_seed(SEED)
-        loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, generator=gen)
-
-        for _ in range(self.k_epochs):
-            for b_ls, b_gs, b_ac, b_lp, b_adv, b_ret in loader:
-                mean, std = self.actor(b_ls)
-                dist = Normal(mean, std)
-
-                entropy = dist.entropy().mean()
-
-                lp_new = dist.log_prob(b_ac).sum(-1)
-                ratio = torch.exp(lp_new - b_lp)
-                surr1 = ratio * b_adv
-                surr2 = torch.clamp(ratio, 1 - self.clip_eps, 1 + self.clip_eps) * b_adv
-
-                actor_loss = -torch.min(surr1, surr2).mean() - 0.01 * entropy
-
-                self.opt_a.zero_grad()
-                actor_loss.backward()
-                self.opt_a.step()
-
-                val_pred = self.critic(b_gs)
-                critic_loss = nn.MSELoss()(val_pred, b_ret)
-
-                self.opt_c.zero_grad()
-                critic_loss.backward()
-                self.opt_c.step()
-
-        self.step_idx = 0
-        
-    def save(self, path):
-        torch.save({'actor': self.actor.state_dict(),
-                    'critic': self.critic.state_dict()}, path)
-
-    def load(self, path):
-        data = torch.load(path, map_location=device)
-        self.actor.load_state_dict(data['actor'])
-        self.critic.load_state_dict(data['critic'])