Upload models - u0bc3fi9

README.md

@@ -3,14 +3,14 @@ language: en
 library_name: pytorch
 license: mit
 ---
+<!--
+Do not remove elements like the above surrounded by two curly braces and do not add any more of them. These entries are required by the PVNet library and are automaticall infilled when the model is uploaded to huggingface
+-->
 
-
-
-
-
-
+<!-- Title - e.g. PVNet2, WindNet, PVNet India -->
 # PVNet2
 
+<!-- Provide a longer summary of what this model is/does. -->
 ## Model Description
 
 <!-- Provide a longer summary of what this model is/does. -->
@@ -26,11 +26,17 @@ This model class uses satellite data, numerical weather predictions, and recent
 
 ## Data
 
-<!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+<!-- eg.
+The model is trained on data from 2019-2022 and validated on data from 2022-2023. It uses NWP data from ECMWF IFS model, and the UK Met Office UKV model. It uses satellite data from the EUMETSAT MSG SEVIRI instrument.
+
+See the data_config.yaml file for more information on the channels and window-size used for each input data source.
+-->
 
-The model is trained on data from 2019-2022 and validated on data from 2022-2023. See experimental notes in the [the google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing) for more details.
+The model is trained on data from 2019-2021 and validated on data from 2022. It uses NWP data from ECMWF IFS model, and the UK Met Office UKV model. It uses also uses inputs from OCF's cloudcasting model
 
 
+
+<!-- The preprocessing section is not strictly nessessary but perhaps nice to have -->
 ### Preprocessing
 
 Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Dataset [2].
@@ -38,19 +44,17 @@ Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Da
 
 ## Results
 
+<!-- Do not remove the lines below -->
 The training logs for the current model can be found here:
- - [https://wandb.ai/openclimatefix/pvnet2.1/runs/nizugje9](https://wandb.ai/openclimatefix/pvnet2.1/runs/nizugje9)
-
-
-The training logs for all model runs of PVNet2 can be found [here](https://wandb.ai/openclimatefix/pvnet2.1).
-
-Some experimental notes can be found at in [the google doc](https://docs.google.com/document/d/1fbkfkBzp16WbnCg7RDuRDvgzInA6XQu3xh4NCjV-WDA/edit?usp=sharing)
+ - [https://wandb.ai/openclimatefix/pvnet2.1/runs/u0bc3fi9](https://wandb.ai/openclimatefix/pvnet2.1/runs/u0bc3fi9)
 
 
+<!-- The hardware section is also just nice to have -->
 ### Hardware
 
 Trained on a single NVIDIA Tesla T4
 
+<!-- Do not remove the section below -->
 ### Software
 
 This model was trained using the following Open Climate Fix packages:
@@ -58,10 +62,7 @@ This model was trained using the following Open Climate Fix packages:
 - [1] https://github.com/openclimatefix/PVNet
 - [2] https://github.com/openclimatefix/ocf-data-sampler
 
+<!-- Especially do not change the two lines below -->
 The versions of these packages can be found below:
- - pvnet==4.1.6
- - ocf-data-sampler==0.2.10
-
-
----
-**Migration Note**: This model was migrated on 2025-08-08 to pvnet version 5.0.3.post0+git.c5a17176.dirty
+ - pvnet==5.0.0.post1+git.f4f6bfed.dirty
+ - ocf-data-sampler==0.3.1

data_config.yaml

@@ -1,6 +1,5 @@
 general:
-  description: Config for training the saved PVNet model
-  name: PVNet current
+  description: Config for producing batches for training PVNet+cloudcasting
 input_data:
   gsp:
     boundaries_version: '20250109'
@@ -11,6 +10,73 @@ input_data:
     time_resolution_minutes: 30
     zarr_path: PLACEHOLDER.zarr
   nwp:
+    ecmwf:
+      accum_channels:
+      - dswrf
+      - dlwrf
+      - sr
+      - duvrs
+      channels:
+      - t2m
+      - dswrf
+      - dlwrf
+      - hcc
+      - mcc
+      - lcc
+      - tcc
+      - sd
+      - sr
+      - duvrs
+      - u10
+      - v10
+      dropout_fraction: 1.0
+      dropout_timedeltas_minutes:
+      - -360
+      image_size_pixels_height: 12
+      image_size_pixels_width: 12
+      interval_end_minutes: 480
+      interval_start_minutes: -120
+      max_staleness_minutes: null
+      normalisation_constants:
+        diff_dlwrf:
+          mean: 1136464.0
+          std: 131942.03125
+        diff_dswrf:
+          mean: 420584.6875
+          std: 715366.3125
+        diff_duvrs:
+          mean: 48265.4765625
+          std: 81605.25
+        diff_sr:
+          mean: 469169.5
+          std: 818950.6875
+        hcc:
+          mean: 0.3961029052734375
+          std: 0.42244860529899597
+        lcc:
+          mean: 0.44901806116104126
+          std: 0.3791404366493225
+        mcc:
+          mean: 0.3288780450820923
+          std: 0.38039860129356384
+        sd:
+          mean: 8.107526082312688e-05
+          std: 0.000913831521756947
+        t2m:
+          mean: 283.48333740234375
+          std: 3.692270040512085
+        tcc:
+          mean: 0.7049227356910706
+          std: 0.37487083673477173
+        u10:
+          mean: 1.7677178382873535
+          std: 5.531515598297119
+        v10:
+          mean: 0.985887885093689
+          std: 5.411230564117432
+      provider: ecmwf
+      time_resolution_minutes: 60
+      zarr_path: PLACEHOLDER.zarr
     ukv:
       channels:
       - t
@@ -69,6 +135,63 @@ input_data:
       provider: ukv
       time_resolution_minutes: 60
       zarr_path: PLACEHOLDER.zarr
+    sat_pred:
+      provider: cloudcasting
+      zarr_path: PLACEHOLDER.zarr
+      interval_start_minutes: 15
+      interval_end_minutes: 180
+      time_resolution_minutes: 15
+      image_size_pixels_height: 24
+      image_size_pixels_width: 24
+      dropout_timedeltas_minutes: []
+      dropout_fraction: 0
+      max_staleness_minutes: null
+      channels:
+      - IR_016
+      - IR_039
+      - IR_087
+      - IR_097
+      - IR_108
+      - IR_120
+      - IR_134
+      - VIS006
+      - VIS008
+      - WV_062
+      - WV_073
+      normalisation_constants:
+        IR_016:
+          mean: 0.17594202
+          std: 0.21462157
+        IR_039:
+          mean: 0.86167645
+          std: 0.04618041
+        IR_087:
+          mean: 0.7719318
+          std: 0.06687243
+        IR_097:
+          mean: 0.8014212
+          std: 0.0468558
+        IR_108:
+          mean: 0.71254843
+          std: 0.17482725
+        IR_120:
+          mean: 0.89058584
+          std: 0.06115861
+        IR_134:
+          mean: 0.944365
+          std: 0.04492306
+        VIS006:
+          mean: 0.09633306
+          std: 0.12184761
+        VIS008:
+          mean: 0.11426069
+          std: 0.13090034
+        WV_062:
+          mean: 0.7359355
+          std: 0.16111417
+        WV_073:
+          mean: 0.62479186
+          std: 0.12924142
   solar_position:
     interval_end_minutes: 480
     interval_start_minutes: -120

model_config.yaml

@@ -7,6 +7,7 @@ output_quantiles:
 - 0.75
 - 0.9
 - 0.98
+add_image_embedding_channel: false
 nwp_encoders_dict:
   ukv:
     _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
@@ -16,8 +17,26 @@ nwp_encoders_dict:
     number_of_conv3d_layers: 6
     conv3d_channels: 32
     image_size_pixels: 24
+  ecmwf:
+    _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
+    _partial_: true
+    in_channels: 12
+    out_features: 256
+    number_of_conv3d_layers: 4
+    conv3d_channels: 32
+    image_size_pixels: 12
+  sat_pred:
+    _target_: pvnet.models.late_fusion.encoders.encoders3d.ResConv3DNet
+    _partial_: true
+    in_channels: 11
+    out_features: 256
+    image_size_pixels: 24
+    hidden_channels: 32
+    n_res_blocks: 3
+    res_block_layers: 3
+    batch_norm: true
+    dropout_frac: 0.0
 sat_encoder: null
-add_image_embedding_channel: false
 pv_encoder: null
 output_network:
   _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
@@ -31,13 +50,18 @@ include_sun: true
 include_gsp_yield_history: false
 forecast_minutes: 480
 history_minutes: 120
-min_sat_delay_minutes: 30
-sat_history_minutes: 60
-pv_history_minutes: 180
 nwp_history_minutes:
   ukv: 120
+  ecmwf: 120
+  sat_pred: -15
 nwp_forecast_minutes:
   ukv: 480
+  ecmwf: 480
+  sat_pred: 180
+nwp_interval_minutes:
+  ukv: 60
+  ecmwf: 60
+  sat_pred: 15
 location_id_mapping:
   1: 1
   2: 2
@@ -370,4 +394,4 @@ location_id_mapping:
   340: 329
   341: 330
   342: 331
-adapt_batches: true
+adapt_batches: false

model_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d71121e13487c529f36a4cb4b7c215ece35a665d662e14d836b7332bc2c478d
-size 27734920
+oid sha256:1e4c41f2ed9186132832d1e15cdb96062248180f9da828cff8bb837ea1678008
+size 35975432