Repository: electricitymaps/electricitymaps-contrib

Issue:
[feature in development] Regional Emission Factors 🗺️ 🔨
<div align="center">
	<img width="589" alt="Regional emission factors" src="https://user-images.githubusercontent.com/3296643/199442712-4ea9d2c6-26bd-43ad-b199-91f02277889c.png">
</div>

### Description
**tl;dr: We're currently implementing regional emission factors!**

Since the dawn of Electricity Maps (or very close to it at least), we have had one grand issue that kept reappearing. It has taken a while, but now we finally have the infrastructure and team in place to work on the this issue: [#738 Refine GHG emission factors by regions](https://github.com/electricitymaps/electricitymaps-contrib/issues/738) 🌍 

Today we mostly use emission factors ([wiki page documenting usage](https://github.com/electricitymaps/electricitymaps-contrib/wiki/Default-emission-factors)) established by the IPCC in 2014, which is not ideal because they are both outdated and do not convey regional differences for the same production modes. With this new feature in place, we will be able to use regional data from reliable sources to give a more accurate estimate for the CO2 impact of different production modes.

All calculations and sources for these regional emission factors will be open and transparent - we'll very soon implement the first few in this repository, so stay tuned! 🔜 

### Timeline

- Right now, the team is working behind the scenes on making this possible - we're aiming to have a first version out before end of November 2022
- At first we will focus on US and a few EU zones - but we're building an open, scalable system that means everyone can contribute with new data for new zones!
- Once it's launched and working, we will document how to contribute and work on making it as easy as possible to gather data, update regions and build the best possible map together with you 💚 

**We will share updates on this issue whenever there's anything new.**

_Lastly, we decided to keep conversations about this topic in a new issue here instead of continuing in #738, as the conversations there were focused on specific data sources, while we here want to talk about the functionality itself_


Discussion:

Resolving patch:
diff --git a/electricitymap/contrib/config/model.py b/electricitymap/contrib/config/model.py
--- a/electricitymap/contrib/config/model.py
+++ b/electricitymap/contrib/config/model.py
@@ -82,6 +82,10 @@ class Parsers(ParsersBaseModel):
     productionPerUnit: Optional[str]
 
 
+class Source(StrictBaseModel):
+    link: str
+
+
 class Delays(StrictBaseModel):
     consumption: Optional[PositiveInt]
     consumptionForecast: Optional[PositiveInt]
@@ -108,6 +112,7 @@ class Zone(StrictBaseModelWithAlias):
     timezone: Optional[str]
     key: ZoneKey  # This is not part of zones/{zone_key}.yaml, but added here to enable self referencing
     estimation_method: Optional[str]
+    sources: Optional[Dict[str, Source]]
 
     def neighbors(self) -> List[ZoneKey]:
         return ZONE_NEIGHBOURS.get(self.key, [])
diff --git a/scripts/update_data_sources.py b/scripts/update_data_sources.py
new file mode 100644
--- /dev/null
+++ b/scripts/update_data_sources.py
@@ -0,0 +1,135 @@
+""" This script aims at automatically updating the data sources file with sources listed in the zones config. """
+import logging
+from collections import namedtuple
+from copy import copy
+from pathlib import Path
+from typing import Dict, List, Set
+
+import arrow
+import pandas as pd
+import yaml
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+CONFIG_DIR = Path(__file__).parent.parent.joinpath("config").resolve()
+EMISSION_FACTORS_SOURCES_FILENAME = (
+    Path(__file__).parent.parent.joinpath("EMISSION_FACTORS_SOURCES.md").resolve()
+)
+
+MD_CONTENT_HEADER = """
+# Emission factors sources
+
+This file describes data sources used for generating the emission factors for all zones.
+
+It only describes zone specific emission factors. Our default emission factors come  from the [IPCC (2014) Fith Assessment Report](https://www.ipcc.ch/site/assets/uploads/2018/02/ipcc_wg3_ar5_annex-iii.pdf#page=7) report, and are fully described in our [wiki](https://github.com/electricitymaps/electricitymaps-contrib/wiki/Default-emission-factors).
+
+## Zone specific emission factors
+
+&nbsp;<details><summary>Click to see the full list of sources</summary>
+
+"""
+
+
+def _find_emission_factor_sources(
+    zone_config: dict,
+) -> Dict[str, Dict[str, Dict[str, str]]]:
+    zone_sources = zone_config.get("sources", {})
+
+    def _get_sources_for_type(_type: str) -> Dict[str, Dict[str, str]]:
+        sources = {}
+        for mode, ef in zone_config.get("emissionFactors", {}).get(_type, {}).items():
+            sources_per_mode = {}
+            if isinstance(ef, list):
+                for _ef in ef:
+                    for s in zone_sources:
+                        if s in _ef.get("source"):
+                            sources_per_mode[s] = zone_sources[s].get("link")
+            else:
+                for s in zone_sources:
+                    if s in ef.get("source"):
+                        sources_per_mode[s] = zone_sources[s].get("link")
+            if sources_per_mode != {}:
+                sources[mode] = sources_per_mode
+        return sources
+
+    sources = {
+        "direct": _get_sources_for_type("direct"),
+        "lifecycle": _get_sources_for_type("lifecycle"),
+    }
+    if sources["direct"] == {}:
+        del sources["direct"]
+    if sources["lifecycle"] == {}:
+        del sources["lifecycle"]
+    return sources
+
+
+def read_zone_config(zone_key: str) -> dict:
+    with open(CONFIG_DIR.joinpath(f"zones/{zone_key}.yaml")) as f:
+        return yaml.safe_load(f)
+
+
+def update_data_sources() -> None:
+    all_emission_factor_sources = {}
+
+    for zone_key in sorted(CONFIG_DIR.joinpath("zones").glob("*.yaml")):
+        zone_key = zone_key.stem
+        zone_config = read_zone_config(zone_key)
+        all_emission_factor_sources[zone_key] = _find_emission_factor_sources(
+            zone_config
+        )
+
+    # Filter out empty sources
+    all_emission_factor_sources = {
+        k: v
+        for k, v in all_emission_factor_sources.items()
+        if v.get("direct", {}) != {} or v.get("lifecycle", {}) != {}
+    }
+
+    md_content = copy(MD_CONTENT_HEADER)
+
+    for zone_key, sources in all_emission_factor_sources.items():
+        zone_content = f"""
+* {zone_key}
+        """
+        if "direct" in sources:
+            zone_content += """
+  * Direct emission factors
+            """
+            for mode, mode_sources in sources["direct"].items():
+                zone_content += f"""
+    * {mode}
+                """
+                for source, link in mode_sources.items():
+                    # We must be careful to not add ";" in the sources
+                    for i, _s in enumerate(source.split("; ")):
+                        zone_content += f"""
+      * [{_s}]({link.split(', ')[i]})
+                """
+        if "lifecycle" in sources:
+            zone_content += """
+  * Lifecycle emission factors
+            """
+            for mode, mode_sources in sources["lifecycle"].items():
+                zone_content += f"""
+    * {mode}
+                """
+                for source, link in mode_sources.items():
+                    # We must be careful to not add ";" in the sources
+                    for i, _s in enumerate(source.split("; ")):
+                        zone_content += f"""
+      * [{_s}]({link.split(', ')[i]})
+                """
+        md_content += zone_content
+
+    md_content += """
+
+    &nbsp;</details>
+    """
+
+    with open(EMISSION_FACTORS_SOURCES_FILENAME, "w") as f:
+        f.write(md_content)
+
+
+if __name__ == "__main__":
+    update_data_sources()
diff --git a/scripts/update_emission_factors_sources.py b/scripts/update_emission_factors_sources.py
new file mode 100644
--- /dev/null
+++ b/scripts/update_emission_factors_sources.py
@@ -0,0 +1,132 @@
+""" This script aims at automatically updating the data sources file with sources listed in the zones config. """
+import logging
+from copy import copy
+from pathlib import Path
+from typing import Dict
+
+import yaml
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+CONFIG_DIR = Path(__file__).parent.parent.joinpath("config").resolve()
+EMISSION_FACTORS_SOURCES_FILENAME = (
+    Path(__file__).parent.parent.joinpath("EMISSION_FACTORS_SOURCES.md").resolve()
+)
+
+MD_CONTENT_HEADER = """
+# Emission factors sources
+
+This file describes data sources used for generating the emission factors for all zones.
+
+It only describes zone specific emission factors. Our default emission factors come  from the [IPCC (2014) Fith Assessment Report](https://www.ipcc.ch/site/assets/uploads/2018/02/ipcc_wg3_ar5_annex-iii.pdf#page=7) report, and are fully described in our [wiki](https://github.com/electricitymaps/electricitymaps-contrib/wiki/Default-emission-factors).
+
+## Zone specific emission factors
+
+&nbsp;<details><summary>Click to see the full list of sources</summary>
+
+"""
+
+
+def _find_emission_factor_sources(
+    zone_config: dict,
+) -> Dict[str, Dict[str, Dict[str, str]]]:
+    zone_sources = zone_config.get("sources", {})
+
+    def _get_sources_for_type(_type: str) -> Dict[str, Dict[str, str]]:
+        sources = {}
+        for mode, ef in zone_config.get("emissionFactors", {}).get(_type, {}).items():
+            sources_per_mode = {}
+            if isinstance(ef, list):
+                for _ef in ef:
+                    for s in zone_sources:
+                        if s in _ef.get("source"):
+                            sources_per_mode[s] = zone_sources[s].get("link")
+            else:
+                for s in zone_sources:
+                    if s in ef.get("source"):
+                        sources_per_mode[s] = zone_sources[s].get("link")
+            if sources_per_mode != {}:
+                sources[mode] = sources_per_mode
+        return sources
+
+    sources = {
+        "direct": _get_sources_for_type("direct"),
+        "lifecycle": _get_sources_for_type("lifecycle"),
+    }
+    if sources["direct"] == {}:
+        del sources["direct"]
+    if sources["lifecycle"] == {}:
+        del sources["lifecycle"]
+    return sources
+
+
+def read_zone_config(zone_key: str) -> dict:
+    with open(CONFIG_DIR.joinpath(f"zones/{zone_key}.yaml")) as f:
+        return yaml.safe_load(f)
+
+
+def update_data_sources() -> None:
+    all_emission_factor_sources = {}
+
+    for zone_key in sorted(CONFIG_DIR.joinpath("zones").glob("*.yaml")):
+        zone_key = zone_key.stem
+        zone_config = read_zone_config(zone_key)
+        all_emission_factor_sources[zone_key] = _find_emission_factor_sources(
+            zone_config
+        )
+
+    # Filter out empty sources
+    all_emission_factor_sources = {
+        k: v
+        for k, v in all_emission_factor_sources.items()
+        if v.get("direct", {}) != {} or v.get("lifecycle", {}) != {}
+    }
+
+    md_content = copy(MD_CONTENT_HEADER)
+
+    for zone_key, sources in all_emission_factor_sources.items():
+        zone_content = f"""
+* {zone_key}
+        """
+        if "direct" in sources:
+            zone_content += """
+  * Direct emission factors
+            """
+            for mode, mode_sources in sources["direct"].items():
+                zone_content += f"""
+    * {mode}
+                """
+                for source, link in mode_sources.items():
+                    # We must be careful to not add ";" in the sources
+                    for i, _s in enumerate(source.split("; ")):
+                        zone_content += f"""
+      * [{_s}]({link.split(', ')[i]})
+                """
+        if "lifecycle" in sources:
+            zone_content += """
+  * Lifecycle emission factors
+            """
+            for mode, mode_sources in sources["lifecycle"].items():
+                zone_content += f"""
+    * {mode}
+                """
+                for source, link in mode_sources.items():
+                    # We must be careful to not add ";" in the sources
+                    for i, _s in enumerate(source.split("; ")):
+                        zone_content += f"""
+      * [{_s}]({link.split(', ')[i]})
+                """
+        md_content += zone_content
+
+    md_content += """
+
+    &nbsp;</details>
+    """
+
+    with open(EMISSION_FACTORS_SOURCES_FILENAME, "w") as f:
+        f.write(md_content)
+
+
+if __name__ == "__main__":
+    update_data_sources()


Test patch:
diff --git a/parsers/test/test_config.py b/parsers/test/test_config.py
--- a/parsers/test/test_config.py
+++ b/parsers/test/test_config.py
@@ -35,18 +35,18 @@ def test_emission_factors(self):
         expected = {
             "battery charge": 0,
             "battery discharge": 54.19088892903222,
-            "biomass": 230,
-            "coal": 820,
-            "gas": 490,
+            "biomass": 819.5789414,
+            "coal": 953.9335274,
+            "gas": 624.846764,
             "geothermal": 38,
-            "hydro": 24,
+            "hydro": 10.7,
             "hydro charge": 0,
             "hydro discharge": 54.19088892903222,
-            "nuclear": 12,
-            "oil": 650,
-            "solar": 45,
+            "nuclear": 5.13,
+            "oil": 1013.5956561,
+            "solar": 30.075,
             "unknown": 700,
-            "wind": 11,
+            "wind": 12.62,
         }
         self.assertEqual(emission_factors("FR"), expected)  # type: ignore