Repository: globaldothealth/list

Issue:
Geocoding issues with DistritoFederal parser
There are some geocoding issues with the Distrito Federal parser which mean that certain cases are being geocoded to other Brazilian states (specifically with the most recent upload it is Parana and Espirito Santo). As you can see in the below screenshot it seems in both cases it is geocoding it to 'Rua Distrito Federal', so using Distrito Federal as a street name rather than the state (ignore the weird symbols this is because of Windows encoding).

![image](https://user-images.githubusercontent.com/62887048/101765198-e8528080-3ad8-11eb-8cfa-87dd6824bc56.png)

The parser is set up to only ingest cases where state == ESPIRITO SANTO so this is not the issue. I have also looked at all the individual cases where the geocoding is failing to see if I can spot a pattern but so far I haven't found anything. For instance, for the second to last Parana case in the above screenshot, if the data is filtered by the case information it could correspond to any of the 4 following cases

![image](https://user-images.githubusercontent.com/62887048/101765454-3c5d6500-3ad9-11eb-84fd-e8b3ae12bbdc.png)

As you can see there is no common feature, I thought it may occur because RA (municipality) was unpopulated but this is not the case.

As I was going through this I remembered a conversation I had with AB and Tim a while back about this which I think is related; essentially it seems some (or most, in fact) municipalities in Distrito Federal are not in Mapbox, see for instance Lago Sul. For these locations, there is only a match up until Distrito Federal (this may have something to do with the unique administrative organization of the DF state see https://en.wikipedia.org/wiki/Administrative_regions_of_the_Federal_District_(Brazil)). So I'm wondering if the unrecognized municipalities are confusing Mapbox, making it take Distrito Federal as a street name in some cases, and therefore returning the locations in the other states. 

I think the only solution to this is that we basically set the query as 'Distrito Federal, Brazil' for all cases, and include the provided municipality info in the notes. Do you agree?

 




Comments:

Patch:
diff --git a/ingestion/functions/parsing/brazil_distrito_federal/dictionaries.json b/ingestion/functions/parsing/brazil_distrito_federal/dictionaries.json
new file mode 100644
index 00000000..abc696e8
--- /dev/null
+++ b/ingestion/functions/parsing/brazil_distrito_federal/dictionaries.json
@@ -0,0 +1,143 @@
+{
+    "comorbidities": {
+        "Pneumopatia": "lung disease",
+        "Nefropatia": "kidney disease",
+        "Distúrbios Metabólicos": "disease of metabolism",
+        "Obesidade": "obesity",
+        "Cardiovasculopatia": "cardiovascular system disease"
+    },
+    "admin_districts": {
+        "Plano Piloto": {
+            "latitude": -15.7939,
+            "longitude": -47.8828
+        },
+        "Gama": {
+            "latitude": -16.0194,
+            "longitude": -48.0669
+        },
+        "Taguatinga": {
+            "latitude": -15.8333,
+            "longitude": -48.0564
+        },
+        "Brazlândia": {
+            "latitude": -15.675,
+            "longitude": -48.2008
+        },
+        "Sobradinho": {
+            "latitude": -15.6531,
+            "longitude": -47.7914
+        },
+        "Planaltina": {
+            "latitude": -15.6192,
+            "longitude": -47.6525
+        },
+        "Paranoá": {
+            "latitude": -15.7686,
+            "longitude": -47.7797
+        },
+        "Núcleo Bandeirante": {
+            "latitude": -15.87,
+            "longitude": -47.9678
+        },
+        "Ceilândia": {
+            "latitude": -15.8192,
+            "longitude": -48.1083
+        },
+        "Guará": {
+            "latitude": -15.8256,
+            "longitude": -47.98
+        },
+        "Cruzeiro": {
+            "latitude": -15.7908,
+            "longitude": -47.9372
+        },
+        "Samambaia": {
+            "latitude": -15.8761,
+            "longitude": -48.0886
+        },
+        "Santa Maria": {
+            "latitude": -16.0153,
+            "longitude": -48.0131
+        },
+        "São Sebastião": {
+            "latitude": -15.9006,
+            "longitude": -47.7794
+        },
+        "Recanto das Emas": {
+            "latitude": -15.9081,
+            "longitude": -48.0731
+        },
+        "Lago Sul": {
+            "latitude": -15.8428,
+            "longitude": -47.8778
+        },
+        "Riacho Fundo": {
+            "latitude": -15.8833,
+            "longitude": -48.0175
+        },
+        "Lago Norte": {
+            "latitude": -15.7386,
+            "longitude": -47.8594
+        },
+        "Candangolândia": {
+            "latitude": -15.8544,
+            "longitude": -47.95
+        },
+        "Águas Claras": {
+            "latitude": -15.8403,
+            "longitude": -48.0278
+        },
+        "Riacho Fundo II": {
+            "latitude": -15.8954,
+            "longitude": -48.0494
+        },
+        "Sudoeste/Octogonal": {
+            "latitude": -15.8055,
+            "longitude": -47.9429
+        },
+        "Varjão": {
+            "latitude": -15.7097,
+            "longitude": -47.8789
+        },
+        "Park Way": {
+            "latitude": -15.9070,
+            "longitude": -47.9647
+        },
+        "SCIA": {
+            "latitude": -15.7875,
+            "longitude": -47.9786
+        },
+        "Sobradinho II": {
+            "latitude": -15.6549,
+            "longitude": -47.7931
+        },
+        "Jardim Botânico": {
+            "latitude": -15.8667,
+            "longitude": -47.7892
+        },
+        "Itapoã": {
+            "latitude": -15.7483,
+            "longitude": -47.7689
+        },
+        "SIA": {
+            "latitude": -15.8011,
+            "longitude": -47.9506
+        },
+        "Vicente Pires": {
+            "latitude": -15.8033,
+            "longitude": -48.03
+        },
+        "Fercal": {
+            "latitude": -15.6028,
+            "longitude": -47.8708
+        },
+        "Pôr do Sol": {
+            "latitude": -15.8553,
+            "longitude": -48.1113
+        },
+        "Arniqueira": {
+            "latitude": -15.8579,
+            "longitude": -48.0046
+        }
+    }
+}
\ No newline at end of file
diff --git a/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal.py b/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal.py
index 63e3b700..303d6553 100644
--- a/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal.py
+++ b/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal.py
@@ -2,6 +2,7 @@ import os
 import sys
 from datetime import datetime
 import csv
+import json
 
 # Layer code, like parsing_lib, is added to the path by AWS.
 # To test locally (e.g. via pytest), we have to modify sys.path.
@@ -32,13 +33,13 @@ _OBESITY = "Obesidade"
 _OTHERS = "Outros"
 _CARDIOVASCULAR = "Cardiovasculopatia"
 
-_COMORBIDITIES_MAP = {
-    "Pneumopatia": "lung disease",
-    "Nefropatia": "kidney disease",
-    "Distúrbios Metabólicos": "disease of metabolism",
-    "Obesidade": "obesity",
-    "Cardiovasculopatia": "cardiovascular system disease"
-}
+
+with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "dictionaries.json")) as json_file:
+    dictionaries = json.load(json_file)
+
+_COMORBIDITIES_MAP = dictionaries["comorbidities"]
+
+_ADMINISTRATIVE_DISTRICTS_MAP = dictionaries["admin_districts"]
 
 
 def convert_date(raw_date):
@@ -132,7 +133,7 @@ def convert_demographics(gender: str, age: str):
         elif age == "<= 19 anos":
             demo["ageRange"] = {"start": 0, "end": 19}
         else:
-        # Age in format '20 a 29 anos'
+            # Age in format '20 a 29 anos'
             age_range = age.partition(" a ")
             demo["ageRange"] = {"start": float(age_range[0]), "end": float("".join([i for i in age_range[2] if not i.isalpha()]))}
     return demo
@@ -151,6 +152,29 @@ def convert_notes(hematologic: str, immunosuppressed: str, other: str):
         return (", ").join(raw_notes)
 
 
+def convert_location(municipality: str):
+    location = {}
+    geometry = {}
+    if municipality and municipality in _ADMINISTRATIVE_DISTRICTS_MAP.keys():
+        location["place"] = municipality
+        location["country"] = "Brazil"
+        location["administrativeAreaLevel1"] = "Distrito Federal"
+        location["geoResolution"] = "Point"
+        location["name"] = str(municipality + ", Distrito Federal, Brazil")
+        
+        geometry["latitude"] = _ADMINISTRATIVE_DISTRICTS_MAP[municipality]["latitude"]
+        geometry["longitude"] = _ADMINISTRATIVE_DISTRICTS_MAP[municipality]["longitude"]
+        location["geometry"] = geometry
+    else:
+        # In local testing the only unknown administrative districts were empty entries, 'Entorno DF' (== Surroundings of Distrito Federal), and 'Sistema Penitenciário' (== Penitenciary system)
+        print(f'Unknown administrative district: {municipality}')
+        location["query"] = "Distrito Federal, Brazil"
+    if location:
+        return location
+    else:
+        return None
+
+
 def parse_cases(raw_data_file: str, source_id: str, source_url: str):
     """
     Parses G.h-format case data from raw API data.
@@ -164,11 +188,7 @@ def parse_cases(raw_data_file: str, source_id: str, source_url: str):
                 try:
                     case = {
                         "caseReference": {"sourceId": source_id, "sourceUrl": source_url},
-                        "location": {
-                            "query": ", ".join(
-                                [row[_MUNICIPALITY], "Distrito Federal", "Brazil"]
-                            )
-                        },
+                        "location": convert_location(row[_MUNICIPALITY]),
                         "events": convert_events(
                             row[_DATE_CONFIRMED],
                             row[_DATE_SYMPTOMS],
diff --git a/ingestion/functions/parsing/brazil_distrito_federal/input_event.json b/ingestion/functions/parsing/brazil_distrito_federal/input_event.json
index 15c028c1..ecb6c973 100644
--- a/ingestion/functions/parsing/brazil_distrito_federal/input_event.json
+++ b/ingestion/functions/parsing/brazil_distrito_federal/input_event.json
@@ -1,7 +1,7 @@
 {
     "env": "local",
     "s3Bucket": "epid-sources-raw",
-    "s3Key": "5f6db1abbebdca003155fc6e/2020/09/25/1343/content.csv",
-    "sourceUrl": "https://covid19.ssp.df.gov.br/resources/dados/dados-abertos.csv?param=[random]",
-    "sourceId": "5f6db1abbebdca003155fc6e"
+    "s3Key": "5ff34f6a5556a3004c58a270/2021/01/04/1729/content.csv",
+    "sourceUrl": "https://covid19.ssp.df.gov.br/resources/dados/dados-abertos.csv",
+    "sourceId": "5ff34f6a5556a3004c58a270"
 }
\ No newline at end of file


Test patch:
diff --git a/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal_test.py b/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal_test.py
index d1883403..df121661 100644
--- a/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal_test.py
+++ b/ingestion/functions/parsing/brazil_distrito_federal/distrito_federal_test.py
@@ -9,7 +9,17 @@ _PARSED_CASE = ({
         "sourceId": "abc123",
         "sourceUrl": "https://covid19.ssp.df.gov.br/resources/dados/dados-abertos.csv?param=[random]",
     },
-    "location": {"query": "Lago Sul, Distrito Federal, Brazil"},
+    "location": {
+        "place": "Lago Sul",
+        "country": "Brazil",
+        "administrativeAreaLevel1": "Distrito Federal",
+        "geoResolution": "Point",
+        "name": "Lago Sul, Distrito Federal, Brazil",
+        "geometry": {
+            "latitude": -15.8428,
+            "longitude": -47.8778
+        }
+    },
     "events": [
         {
             "name": "confirmed",