Upload model - 9qyz1dl8

README.md

@@ -3,18 +3,18 @@ language: en
 library_name: pytorch
 license: mit
 ---
+<!--
+Do not remove elements like the above surrounded by two curly braces and do not add any more of them. These entries are required by the PVNet library and are automaticall infilled when the model is uploaded to huggingface
+-->
 
+<!-- Title - e.g. PVNet2, WindNet, PVNet India -->
+# PVNet2
 
-
-
-
-
-# PVNet_summation
-
+<!-- Provide a longer summary of what this model is/does. -->
 ## Model Description
 
 <!-- Provide a longer summary of what this model is/does. -->
-This model class sums the output of the PVNet model's GSP level predictions to make a national forecast of UK PV output. More information can be found in the model repo [1], the PVNet model repo [2], and experimental notes in [this google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing).
+This model class uses satellite data, and numerical weather predictions to forecast the near-term (~8 hours) PV power output at all GSPs. More information can be found in the model repo [1].
 
 - **Developed by:** openclimatefix
 - **Model type:** Fusion model
@@ -26,38 +26,44 @@ This model class sums the output of the PVNet model's GSP level predictions to m
 
 ## Data
 
-<!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+<!-- eg.
+The model is trained on data from 2019-2022 and validated on data from 2022-2023. It uses NWP data from ECMWF IFS model, and the UK Met Office UKV model. It uses satellite data from the EUMETSAT MSG SEVIRI instrument.
+
+See the data_config.yaml file for more information on the channels and window-size used for each input data source.
+-->
 
-The model is trained on the output predictions of our PVNet model which gives GSP (i.e. UK regional) level predictions
-of solar power across Great Britain. This model is trained to take those predictions and use them to estimate
-the national total with uncertainty estimates.
+The model is trained on data from 2019-2021 and validated on data from 2022. It uses NWP data from ECMWF IFS model, and the UK Met Office UKV model. It uses also uses inputs from OCF's cloudcasting model
 
 
+
+<!-- The preprocessing section is not strictly nessessary but perhaps nice to have -->
 ### Preprocessing
 
-The input data is prepared with the `ocf_data_sampler` [3].
+Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Dataset [2].
 
 
 ## Results
 
+<!-- Do not remove the lines below -->
 The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/pvnet_summation/runs/j9c7l2ob](https://wandb.ai/openclimatefix/pvnet_summation/runs/j9c7l2ob)
-
-
-The training logs for all model runs of PVNet_summation can be found [here](https://wandb.ai/openclimatefix/pvnet_summation).
-
-Some experimental notes can be found at in [the google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing)
 
 
-### Hardware
+<!-- The hardware section is also just nice to have -->
+<!-- ### Hardware 
 
 Trained on a single NVIDIA Tesla T4
 
+-->
+
+<!-- Do not remove the section below -->
 ### Software
 
-- [1] https://github.com/openclimatefix/PVNet_summation
-- [2] https://github.com/openclimatefix/PVNet
-- [3] https://github.com/openclimatefix/ocf-data-sampler
+This model was trained using the following Open Climate Fix packages:
 
----
-**Migration Note**: This model was migrated on 2025-08-08 to pvnet-summation version 1.0.0.post0+git.cd7464d2.dirty
+- [1] https://github.com/openclimatefix/PVNet
+- [2] https://github.com/openclimatefix/ocf-data-sampler
+
+<!-- Especially do not change the two lines below -->
+The versions of these packages can be found below:
+ - pvnet_summation==1.1.0.post0+git.3b6f1115.dirty
+ - ocf-data-sampler==0.5.26.post2+git.90ee263d.dirty

datamodule_config.yaml

@@ -1,3 +1,3 @@
 pvnet_model:
   model_id: openclimatefix/pvnet_uk_region_day_ahead
-  revision: e12f7d7c16a00c41b03aecb787a7e992fbd4235b
+  revision: ace3469f6fb6db7356afe401c1aaf1a78505f4f7

full_experiment_config.yaml

@@ -0,0 +1,127 @@
+trainer:
+  _target_: lightning.pytorch.trainer.trainer.Trainer
+  accelerator: gpu
+  devices:
+  - 1
+  min_epochs: null
+  max_epochs: 300
+  reload_dataloaders_every_n_epochs: 0
+  num_sanity_val_steps: 8
+  fast_dev_run: false
+  log_every_n_steps: 50
+model:
+  _target_: pvnet_summation.training.lightning_module.PVNetSummationLightningModule
+  model:
+    _target_: pvnet_summation.models.horizon_dense_model.HorizonDenseModel
+    output_quantiles:
+    - 0.02
+    - 0.1
+    - 0.25
+    - 0.5
+    - 0.75
+    - 0.9
+    - 0.98
+    output_network:
+      _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
+      _partial_: true
+      fc_hidden_features: 128
+      n_res_blocks: 3
+      res_block_layers: 2
+      dropout_frac: 0.2
+    predict_difference_from_sum: true
+    force_non_crossing: true
+    beta: 3
+    use_horizon_encoding: true
+    use_solar_position: true
+    history_minutes: 120
+    forecast_minutes: 2160
+    interval_minutes: 30
+    num_input_locations: 331
+    input_quantiles:
+    - 0.02
+    - 0.1
+    - 0.25
+    - 0.5
+    - 0.75
+    - 0.9
+    - 0.98
+  optimizer:
+    _target_: pvnet_summation.optimizers.AdamWReduceLROnPlateau
+    lr: 3.0e-05
+    weight_decay: 0.1
+    amsgrad: true
+    patience: 10
+    factor: 0.2
+    threshold: 0.0
+datamodule:
+  data_source_paths:
+    gsp: /mnt/storage_u2_30tb_a/ml_training_zarrs/pv/pvlive_gsp_new_boundaries_2019-2025.zarr
+    satellite:
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2019_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2020_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2021_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2022_nonhrv.zarr
+    nwp:
+      ukv:
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ukv_v7/UKV_intermediate_version_7.1.zarr
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ukv_v7/UKV_2021_missing.zarr
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ukv_v7/UKV_2022.zarr
+      ecmwf:
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ecmwf_v3/ECMWF_2019.zarr
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ecmwf_v3/ECMWF_2020.zarr
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ecmwf_v3/ECMWF_2021.zarr
+      - /mnt/storage_u2_30tb_a/ml_training_zarrs/nwp/ecmwf_v3/ECMWF_2022.zarr
+      cloudcasting: /mnt/raphael/fast/cloudcasting/simvp.zarr
+  batch_size: 32
+  num_workers: 16
+  prefetch_factor: 2
+  persistent_workers: true
+  max_num_train_samples: null
+  max_num_val_samples: null
+  pvnet_model:
+    model_id: openclimatefix/pvnet_uk_region_day_ahead
+    revision: ace3469f6fb6db7356afe401c1aaf1a78505f4f7
+  train_period:
+  - null
+  - '2021-12-31'
+  val_period:
+  - '2022-01-01'
+  - '2022-12-31'
+  seed: ${seed}
+  dataset_pickle_dir: /home/james/tmp/summation_dataset_pickles_pvnet_day_ahead
+callbacks:
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: epoch
+  model_summary:
+    _target_: lightning.pytorch.callbacks.ModelSummary
+    max_depth: 3
+  model_checkpoint:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: ${resolve_monitor_loss:${model.model.output_quantiles}}
+    mode: min
+    save_top_k: 1
+    save_last: true
+    every_n_epochs: 1
+    verbose: false
+    filename: epoch={epoch}-step={step}
+    dirpath: checkpoints/${model_name}
+    auto_insert_metric_name: false
+    save_on_train_epoch_end: false
+logger:
+  wandb:
+    _target_: lightning.pytorch.loggers.wandb.WandbLogger
+    project: pvnet_summation
+    name: ${model_name}
+    save_dir: ./
+    offline: false
+    id: null
+    log_model: false
+    prefix: ''
+    job_type: train
+    group: ''
+    tags: []
+sample_save_dir: /home/james/repos/pvnet-summation/presaved_saved_samples
+work_dir: ${hydra:runtime.cwd}
+model_name: pvnet_day_ahead
+seed: 2727831

model_config.yaml

@@ -1,4 +1,4 @@
-_target_: pvnet_summation.models.dense_model.DenseModel
+_target_: pvnet_summation.models.horizon_dense_model.HorizonDenseModel
 output_quantiles:
 - 0.02
 - 0.1
@@ -11,14 +11,18 @@ output_network:
   _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
   _partial_: true
   fc_hidden_features: 128
-  n_res_blocks: 2
+  n_res_blocks: 3
   res_block_layers: 2
-  dropout_frac: 0.1
+  dropout_frac: 0.2
 predict_difference_from_sum: true
-num_input_locations: 331
+force_non_crossing: true
+beta: 3
+use_horizon_encoding: true
+use_solar_position: true
 history_minutes: 120
 forecast_minutes: 2160
 interval_minutes: 30
+num_input_locations: 331
 input_quantiles:
 - 0.02
 - 0.1

model_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1498a14dfc7126ef2e4eb045042d9d912c539b95ea1bd91e184ed2b047ba0d6c
-size 86968264
+oid sha256:f0f6bd8d607aea0010b37213faf23c92cd7104cdd5a8aefcd2706e948f438f08
+size 1605412