Upload 9 files

Agent.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f3cebfdfbd6341b0a8a2cdb3efe0b0f0a2f48484738fcf2cd6ccb103772045e
+size 1110764

Agent/Agent-450918.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f3cebfdfbd6341b0a8a2cdb3efe0b0f0a2f48484738fcf2cd6ccb103772045e
+size 1110764

Agent/Agent-450918.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecfa1fdb2f78547e4aba84e972f73ed8ab82d2161231066d4eefd5d635a78072
+size 7958695

Agent/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecfa1fdb2f78547e4aba84e972f73ed8ab82d2161231066d4eefd5d635a78072
+size 7958695

Agent/events.out.tfevents.1716716337.RICHARD.22948.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a4916aa2a31ee222778cc207ee2087287adb963da574683bca851c21b810a3
+size 209616

README.md

@@ -0,0 +1,41 @@
+---
+library_name: hivex
+original_train_name: WildfireResourceManagement_difficulty_6_task_1_run_id_1_train
+tags:
+- hivex
+- hivex-wildfire-resource-management
+- reinforcement-learning
+- multi-agent-reinforcement-learning
+model-index:
+- name: hivex-WRM-PPO-baseline-task-1-difficulty-6
+  results:
+  - task:
+      type: sub-task
+      name: keep_all
+      task-id: 1
+      difficulty-id: 6
+    dataset:
+      name: hivex-wildfire-resource-management
+      type: hivex-wildfire-resource-management
+    metrics:
+    - type: cumulative_reward
+      value: 258.2725601196289 +/- 98.13659384326716
+      name: Cumulative Reward
+      verified: true
+    - type: collective_performance
+      value: 45.91880645751953 +/- 18.302557856348788
+      name: Collective Performance
+      verified: true
+    - type: individual_performance
+      value: 24.243414211273194 +/- 9.523378885568981
+      name: Individual Performance
+      verified: true
+    - type: reward_for_moving_resources_to_neighbours
+      value: 1.030250072479248 +/- 0.2913765344241285
+      name: Reward for Moving Resources to Neighbours
+      verified: true
+    - type: reward_for_moving_resources_to_self
+      value: 207.9915672302246 +/- 86.63382909716428
+      name: Reward for Moving Resources to Self
+      verified: true
+---

configuration.yaml

@@ -0,0 +1,107 @@
+default_settings: null
+behaviors:
+  Agent:
+    trainer_type: ppo
+    hyperparameters:
+      batch_size: 128
+      buffer_size: 2048
+      learning_rate: 0.0003
+      beta: 0.01
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+      shared_critic: false
+      learning_rate_schedule: linear
+      beta_schedule: linear
+      epsilon_schedule: linear
+    network_settings:
+      normalize: false
+      hidden_units: 512
+      num_layers: 2
+      vis_encode_type: simple
+      memory: null
+      goal_conditioning_type: hyper
+      deterministic: false
+    reward_signals:
+      curiosity:
+        gamma: 0.99
+        strength: 0.02
+        network_settings:
+          normalize: false
+          hidden_units: 256
+          num_layers: 2
+          vis_encode_type: simple
+          memory: null
+          goal_conditioning_type: hyper
+          deterministic: false
+        learning_rate: 0.0003
+        encoding_size: 256
+      extrinsic:
+        gamma: 0.99
+        strength: 1.0
+        network_settings:
+          normalize: false
+          hidden_units: 128
+          num_layers: 2
+          vis_encode_type: simple
+          memory: null
+          goal_conditioning_type: hyper
+          deterministic: false
+    init_path: null
+    keep_checkpoints: 5
+    checkpoint_interval: 500000
+    max_steps: 450000
+    time_horizon: 2048
+    summary_freq: 4500
+    threaded: true
+    self_play: null
+    behavioral_cloning: null
+env_settings:
+  env_path: c:/users/pdsie/documents/hivex/src/hivex/training/baseline/ml_agents/dev_environments/Hivex_WildfireResourceManagement_win
+  env_args: null
+  base_port: 5006
+  num_envs: 1
+  num_areas: 1
+  seed: 5000
+  max_lifetime_restarts: 10
+  restarts_rate_limit_n: 1
+  restarts_rate_limit_period_s: 60
+engine_settings:
+  width: 84
+  height: 84
+  quality_level: 5
+  time_scale: 20
+  target_frame_rate: -1
+  capture_frame_rate: 60
+  no_graphics: true
+environment_parameters:
+  difficulty:
+    curriculum:
+    - value:
+        sampler_type: constant
+        sampler_parameters:
+          seed: 5000
+          value: 6
+      name: difficulty
+      completion_criteria: null
+  task:
+    curriculum:
+    - value:
+        sampler_type: constant
+        sampler_parameters:
+          seed: 5001
+          value: 1
+      name: task
+      completion_criteria: null
+checkpoint_settings:
+  run_id: WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train
+  initialize_from: null
+  load_model: false
+  resume: false
+  force: false
+  train_model: false
+  inference: false
+  results_dir: results
+torch_settings:
+  device: null
+debug: false

run_logs/timers.json

@@ -0,0 +1,459 @@
+{
+    "name": "root",
+    "gauges": {
+        "Agent.Policy.Entropy.mean": {
+            "value": 3.8415136337280273,
+            "min": 3.683974504470825,
+            "max": 4.382774353027344,
+            "count": 100
+        },
+        "Agent.Policy.Entropy.sum": {
+            "value": 17909.13671875,
+            "min": 8347.8271484375,
+            "max": 30308.748046875,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.IndividualResourceCount.mean": {
+            "value": 0.885185268190172,
+            "min": 0.6388889345581885,
+            "max": 0.9000000953674316,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.IndividualResourceCount.sum": {
+            "value": 23.900002241134644,
+            "min": 7.600000619888306,
+            "max": 34.50000246614218,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.RewardforMovingResourcestoNeighbours.mean": {
+            "value": 1.5381198350202154,
+            "min": 0.5687825783259339,
+            "max": 11.5554188269156,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.RewardforMovingResourcestoNeighbours.sum": {
+            "value": 41.529235545545816,
+            "min": 6.244910421597552,
+            "max": 426.053710103035,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.RewardforMovingResourcestoSelf.mean": {
+            "value": 158.69375182964183,
+            "min": 30.772578737250083,
+            "max": 560.591554429796,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.RewardforMovingResourcestoSelf.sum": {
+            "value": 4284.73129940033,
+            "min": 1109.2449626922607,
+            "max": 9362.944366455078,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.CollectivePerformance.mean": {
+            "value": 38.49101257324219,
+            "min": 21.095816877153183,
+            "max": 113.3098898993598,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.CollectivePerformance.sum": {
+            "value": 1039.257339477539,
+            "min": 392.47331619262695,
+            "max": 2044.7774829864502,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.IndividualPerformance.mean": {
+            "value": 19.324099478898226,
+            "min": 10.97053614589903,
+            "max": 63.26357290479872,
+            "count": 100
+        },
+        "Agent.WildfireResourceManagement.IndividualPerformance.sum": {
+            "value": 521.7506859302521,
+            "min": 224.29837799072266,
+            "max": 1079.490504026413,
+            "count": 100
+        },
+        "Agent.Environment.LessonNumber.difficulty.mean": {
+            "value": 0.0,
+            "min": 0.0,
+            "max": 0.0,
+            "count": 100
+        },
+        "Agent.Environment.LessonNumber.difficulty.sum": {
+            "value": 0.0,
+            "min": 0.0,
+            "max": 0.0,
+            "count": 100
+        },
+        "Agent.Environment.LessonNumber.task.mean": {
+            "value": 0.0,
+            "min": 0.0,
+            "max": 0.0,
+            "count": 100
+        },
+        "Agent.Environment.LessonNumber.task.sum": {
+            "value": 0.0,
+            "min": 0.0,
+            "max": 0.0,
+            "count": 100
+        },
+        "Agent.Environment.EpisodeLength.mean": {
+            "value": 171.33333333333334,
+            "min": 112.2,
+            "max": 485.0,
+            "count": 100
+        },
+        "Agent.Environment.EpisodeLength.sum": {
+            "value": 4626.0,
+            "min": 2142.0,
+            "max": 7497.0,
+            "count": 100
+        },
+        "Agent.Step.mean": {
+            "value": 449953.0,
+            "min": 4410.0,
+            "max": 449953.0,
+            "count": 100
+        },
+        "Agent.Step.sum": {
+            "value": 449953.0,
+            "min": 4410.0,
+            "max": 449953.0,
+            "count": 100
+        },
+        "Agent.Policy.CuriosityValueEstimate.mean": {
+            "value": 0.01624756120145321,
+            "min": -0.011678636074066162,
+            "max": 0.1500101536512375,
+            "count": 100
+        },
+        "Agent.Policy.CuriosityValueEstimate.sum": {
+            "value": 0.5036743879318237,
+            "min": -0.38539499044418335,
+            "max": 2.8629181385040283,
+            "count": 100
+        },
+        "Agent.Policy.ExtrinsicValueEstimate.mean": {
+            "value": 68.95470428466797,
+            "min": 5.23118257522583,
+            "max": 131.2893524169922,
+            "count": 100
+        },
+        "Agent.Policy.ExtrinsicValueEstimate.sum": {
+            "value": 2137.595947265625,
+            "min": 204.01611328125,
+            "max": 3464.6044921875,
+            "count": 100
+        },
+        "Agent.Environment.CumulativeReward.mean": {
+            "value": 165.88100088796307,
+            "min": 71.9970401372665,
+            "max": 578.7887431553432,
+            "count": 100
+        },
+        "Agent.Environment.CumulativeReward.sum": {
+            "value": 5142.3110275268555,
+            "min": 2160.249409198761,
+            "max": 9137.956047058105,
+            "count": 100
+        },
+        "Agent.Policy.CuriosityReward.mean": {
+            "value": 0.02520785219366512,
+            "min": 0.016535775563014405,
+            "max": 0.20388935391719526,
+            "count": 100
+        },
+        "Agent.Policy.CuriosityReward.sum": {
+            "value": 0.7814434180036187,
+            "min": 0.3924146108329296,
+            "max": 7.951684802770615,
+            "count": 100
+        },
+        "Agent.Policy.ExtrinsicReward.mean": {
+            "value": 165.88100088796307,
+            "min": 71.9970401372665,
+            "max": 578.7887431553432,
+            "count": 100
+        },
+        "Agent.Policy.ExtrinsicReward.sum": {
+            "value": 5142.3110275268555,
+            "min": 2160.249409198761,
+            "max": 9137.956047058105,
+            "count": 100
+        },
+        "Agent.Losses.PolicyLoss.mean": {
+            "value": 0.06098024267703295,
+            "min": 0.059025466955010444,
+            "max": 0.08179422431830358,
+            "count": 100
+        },
+        "Agent.Losses.PolicyLoss.sum": {
+            "value": 0.06098024267703295,
+            "min": 0.059025466955010444,
+            "max": 0.16358844863660715,
+            "count": 100
+        },
+        "Agent.Losses.ValueLoss.mean": {
+            "value": 497.4683979175709,
+            "min": 69.88140380382538,
+            "max": 1023.7686164155218,
+            "count": 100
+        },
+        "Agent.Losses.ValueLoss.sum": {
+            "value": 497.4683979175709,
+            "min": 69.88140380382538,
+            "max": 2047.5372328310436,
+            "count": 100
+        },
+        "Agent.Policy.LearningRate.mean": {
+            "value": 1.4520995159999864e-06,
+            "min": 1.4520995159999864e-06,
+            "max": 0.000298578000474,
+            "count": 100
+        },
+        "Agent.Policy.LearningRate.sum": {
+            "value": 1.4520995159999864e-06,
+            "min": 1.4520995159999864e-06,
+            "max": 0.000590892003036,
+            "count": 100
+        },
+        "Agent.Policy.Epsilon.mean": {
+            "value": 0.10048399999999998,
+            "min": 0.10048399999999998,
+            "max": 0.199526,
+            "count": 100
+        },
+        "Agent.Policy.Epsilon.sum": {
+            "value": 0.10048399999999998,
+            "min": 0.10048399999999998,
+            "max": 0.396964,
+            "count": 100
+        },
+        "Agent.Policy.Beta.mean": {
+            "value": 5.835159999999957e-05,
+            "min": 5.835159999999957e-05,
+            "max": 0.0099526474,
+            "count": 100
+        },
+        "Agent.Policy.Beta.sum": {
+            "value": 5.835159999999957e-05,
+            "min": 5.835159999999957e-05,
+            "max": 0.019696703600000004,
+            "count": 100
+        },
+        "Agent.Losses.CuriosityForwardLoss.mean": {
+            "value": 0.008985025592631212,
+            "min": 0.00455733379490222,
+            "max": 0.5244236296663681,
+            "count": 100
+        },
+        "Agent.Losses.CuriosityForwardLoss.sum": {
+            "value": 0.008985025592631212,
+            "min": 0.00455733379490222,
+            "max": 0.5244236296663681,
+            "count": 100
+        },
+        "Agent.Losses.CuriosityInverseLoss.mean": {
+            "value": 3.72886719527068,
+            "min": 3.567552768822872,
+            "max": 4.390662242968877,
+            "count": 100
+        },
+        "Agent.Losses.CuriosityInverseLoss.sum": {
+            "value": 3.72886719527068,
+            "min": 3.567552768822872,
+            "max": 8.744959609072117,
+            "count": 100
+        },
+        "Agent.IsTraining.mean": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 100
+        },
+        "Agent.IsTraining.sum": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 100
+        }
+    },
+    "metadata": {
+        "timer_format_version": "0.1.0",
+        "start_time_seconds": "1716716335",
+        "python_version": "3.9.18 (main, Sep 11 2023, 14:09:26) [MSC v.1916 64 bit (AMD64)]",
+        "command_line_arguments": "C:\\Users\\pdsie\\anaconda3\\envs\\mlagents20\\Scripts\\mlagents-learn c:/users/pdsie/documents/hivex/src/hivex/training/baseline/ml_agents/configs/mlagents/tmp/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train.yaml --run-id=WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train --base-port 5006",
+        "mlagents_version": "0.30.0",
+        "mlagents_envs_version": "0.30.0",
+        "communication_protocol_version": "1.5.0",
+        "pytorch_version": "1.7.1+cu110",
+        "numpy_version": "1.21.0",
+        "end_time_seconds": "1716718253"
+    },
+    "total": 1917.4303124,
+    "count": 1,
+    "self": 0.2302324000002045,
+    "children": {
+        "run_training.setup": {
+            "total": 0.05317609999999995,
+            "count": 1,
+            "self": 0.05317609999999995
+        },
+        "TrainerController.start_learning": {
+            "total": 1917.1469038999999,
+            "count": 1,
+            "self": 1.313527699995575,
+            "children": {
+                "TrainerController._reset_env": {
+                    "total": 2.1729645,
+                    "count": 1,
+                    "self": 2.1729645
+                },
+                "TrainerController.advance": {
+                    "total": 1913.5361583000042,
+                    "count": 50104,
+                    "self": 1.172966500002076,
+                    "children": {
+                        "env_step": {
+                            "total": 1912.3631918000021,
+                            "count": 50104,
+                            "self": 1609.3126831999762,
+                            "children": {
+                                "SubprocessEnvManager._take_step": {
+                                    "total": 302.51252550000277,
+                                    "count": 50104,
+                                    "self": 2.0225113000114447,
+                                    "children": {
+                                        "TorchPolicy.evaluate": {
+                                            "total": 300.4900141999913,
+                                            "count": 50104,
+                                            "self": 300.4900141999913
+                                        }
+                                    }
+                                },
+                                "workers": {
+                                    "total": 0.5379831000233426,
+                                    "count": 50104,
+                                    "self": 0.0,
+                                    "children": {
+                                        "worker_root": {
+                                            "total": 1914.389033100022,
+                                            "count": 50104,
+                                            "is_parallel": true,
+                                            "self": 376.17880840001817,
+                                            "children": {
+                                                "steps_from_proto": {
+                                                    "total": 0.00024369999999995784,
+                                                    "count": 1,
+                                                    "is_parallel": true,
+                                                    "self": 0.00011689999999986433,
+                                                    "children": {
+                                                        "_process_rank_one_or_two_observation": {
+                                                            "total": 0.0001268000000000935,
+                                                            "count": 2,
+                                                            "is_parallel": true,
+                                                            "self": 0.0001268000000000935
+                                                        }
+                                                    }
+                                                },
+                                                "UnityEnvironment.step": {
+                                                    "total": 1538.2099810000038,
+                                                    "count": 50104,
+                                                    "is_parallel": true,
+                                                    "self": 4.100638399950185,
+                                                    "children": {
+                                                        "UnityEnvironment._generate_step_input": {
+                                                            "total": 5.618027100040523,
+                                                            "count": 50104,
+                                                            "is_parallel": true,
+                                                            "self": 5.618027100040523
+                                                        },
+                                                        "communicator.exchange": {
+                                                            "total": 1516.8365850000084,
+                                                            "count": 50104,
+                                                            "is_parallel": true,
+                                                            "self": 1516.8365850000084
+                                                        },
+                                                        "steps_from_proto": {
+                                                            "total": 11.654730500004629,
+                                                            "count": 50104,
+                                                            "is_parallel": true,
+                                                            "self": 5.994611400003981,
+                                                            "children": {
+                                                                "_process_rank_one_or_two_observation": {
+                                                                    "total": 5.660119100000648,
+                                                                    "count": 100208,
+                                                                    "is_parallel": true,
+                                                                    "self": 5.660119100000648
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                "trainer_threads": {
+                    "total": 2.1500000002561137e-05,
+                    "count": 1,
+                    "self": 2.1500000002561137e-05,
+                    "children": {
+                        "thread_root": {
+                            "total": 0.0,
+                            "count": 0,
+                            "is_parallel": true,
+                            "self": 0.0,
+                            "children": {
+                                "trainer_advance": {
+                                    "total": 1913.6148742999821,
+                                    "count": 94273,
+                                    "is_parallel": true,
+                                    "self": 3.426484999992226,
+                                    "children": {
+                                        "process_trajectory": {
+                                            "total": 1495.1014801999897,
+                                            "count": 94273,
+                                            "is_parallel": true,
+                                            "self": 1495.1014801999897
+                                        },
+                                        "_update_policy": {
+                                            "total": 415.0869091000002,
+                                            "count": 148,
+                                            "is_parallel": true,
+                                            "self": 205.6351343000038,
+                                            "children": {
+                                                "TorchPPOOptimizer.update": {
+                                                    "total": 209.45177479999637,
+                                                    "count": 10278,
+                                                    "is_parallel": true,
+                                                    "self": 209.45177479999637
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                "TrainerController._save_models": {
+                    "total": 0.1242319000000407,
+                    "count": 1,
+                    "self": 0.01374260000011418,
+                    "children": {
+                        "RLTrainer._checkpoint": {
+                            "total": 0.11048929999992652,
+                            "count": 1,
+                            "self": 0.11048929999992652
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

run_logs/training_status.json

@@ -0,0 +1,35 @@
+{
+    "difficulty": {
+        "lesson_num": 0
+    },
+    "task": {
+        "lesson_num": 0
+    },
+    "Agent": {
+        "checkpoints": [
+            {
+                "steps": 450918,
+                "file_path": "results\\WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train\\Agent\\Agent-450918.onnx",
+                "reward": 203.45566728379993,
+                "creation_time": 1716718252.8599727,
+                "auxillary_file_paths": [
+                    "results\\WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train\\Agent\\Agent-450918.pt"
+                ]
+            }
+        ],
+        "final_checkpoint": {
+            "steps": 450918,
+            "file_path": "results\\WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train\\Agent.onnx",
+            "reward": 203.45566728379993,
+            "creation_time": 1716718252.8599727,
+            "auxillary_file_paths": [
+                "results\\WildfireResourceManagement/train/WildfireResourceManagement_difficulty_6_task_1_run_id_1_train\\Agent\\Agent-450918.pt"
+            ]
+        }
+    },
+    "metadata": {
+        "stats_format_version": "0.3.0",
+        "mlagents_version": "0.30.0",
+        "torch_version": "1.7.1+cu110"
+    }
+}