Repository: alltheplaces/alltheplaces

Issue:
Enforce 2 digit country code
As noted in https://github.com/alltheplaces/alltheplaces/pull/3791#pullrequestreview-1071740124, the spiders are currently inconstant in the country output.

I think we all agree that we should follow [OSM](https://wiki.openstreetmap.org/wiki/Key:addr:country), and use "ISO 3166-1 alpha-2".

I can envision a wrapper function, like `ensure_country_code("United Kingdom")` that returns "GB". I'm leaning towards this being in spider code (and shared parsers like the ld one) not a pipeline.


Discussion:
So I am cooking something up in my forked repo, initial commit:
https://github.com/mjoe999/alltheplaces/commit/ec663801d3a9b03b6536ed4784ed06c3ae51cef3
... just a heads-up in case anyone is doing same.

@Cj-Malone : I see you have ExtractGBPostcodePipeline .... note that geonames country structure has an embedded postcode regex which if generally correct could prove helpful.

@iandees : geonames is a good dependency to pull in as it gives a ton of city names per territory with lat/lng and population. I believe that this is something you have been missing. For some spiders you will want to do the existing centroid thing but then also call with high population cities and a finer lat/lng grid .... higher population means higher desnity. Think Big Macs.

> I see you have ExtractGBPostcodePipeline .... note that geonames country structure has an embedded postcode regex which if generally correct could prove helpful.

I love the idea of expanding the pipeline for more countries. I'm no regex wizard, but I'm not sure `postalcoderegex` from geonames is right for GB, and if it's wrong here, is it wrong for other countries too?
Also in your `to_iso_alpha2_country_code` I'd move `UNHANDLED_COUNTRY_MAPPINGS` a lot higher up so any of our overrides happen first.

Resolving patch:
diff --git a/locations/pipelines.py b/locations/pipelines.py
--- a/locations/pipelines.py
+++ b/locations/pipelines.py
@@ -9,6 +9,7 @@
 
 from scrapy.exceptions import DropItem
 
+from locations.commands.insights import CountryUtils
 from locations.name_suggestion_index import NSI
 
 
@@ -48,6 +49,18 @@ def process_item(self, item, spider):
         return item
 
 
+class CountryCodeCleanUpPipeline(object):
+    def __init__(self):
+        self.country_utils = CountryUtils()
+
+    def process_item(self, item, spider):
+        if country := item.get("country"):
+            if clean_country := self.country_utils.to_iso_alpha2_country_code(country):
+                item["country"] = clean_country
+
+        return item
+
+
 class ExtractGBPostcodePipeline(object):
     def process_item(self, item, spider):
         if item.get("country") == "GB":
@@ -210,7 +223,6 @@ def process_item(self, item, spider):
 
 
 class ApplyNSICategoriesPipeline(object):
-
     nsi = NSI()
 
     wikidata_cache = {}
@@ -231,19 +243,37 @@ def process_item(self, item, spider):
             spider.crawler.stats.inc_value("atp/nsi/brand_missing")
             return item
 
-        match = None
-
         if len(matches) == 1:
             spider.crawler.stats.inc_value("atp/nsi/perfect_match")
-            match = matches[0]
-        else:
-            # TODO: filter multiple matches after we see the stats
-            spider.crawler.stats.inc_value("atp/nsi/multiple_hits")
-            return item
+            return self.apply_tags(matches[0], item)
 
-        if match is None:
-            return item
+        if cc := item.get("country"):
+            matches = self.filter_cc(matches, cc.lower())
+
+            if len(matches) == 1:
+                spider.crawler.stats.inc_value("atp/nsi/cc_match")
+                return self.apply_tags(matches[0], item)
+
+        spider.crawler.stats.inc_value("atp/nsi/match_failed")
+        return item
+
+    def filter_cc(self, matches, cc) -> []:
+        includes = []
+        globals_matches = []
+
+        for match in matches:
+            if cc in match["locationSet"].get("exclude", []):
+                continue
+
+            include = match["locationSet"].get("include", [])
+            if cc in include:
+                includes.append(match)
+            if "001" in include:  # 001 being global in NSI
+                globals_matches.append(match)
+
+        return includes or globals_matches
 
+    def apply_tags(self, match, item):
         extras = item.get("extras", {})
         extras["nsi_id"] = match["id"]
 
diff --git a/locations/settings.py b/locations/settings.py
--- a/locations/settings.py
+++ b/locations/settings.py
@@ -86,6 +86,7 @@
     "locations.pipelines.DuplicatesPipeline": 200,
     "locations.pipelines.ApplySpiderLevelAttributesPipeline": 300,
     "locations.pipelines.ApplySpiderNamePipeline": 350,
+    "locations.pipelines.CountryCodeCleanUpPipeline": 355,
     "locations.pipelines.ExtractGBPostcodePipeline": 400,
     "locations.pipelines.AssertURLSchemePipeline": 500,
     "locations.pipelines.CheckItemPropertiesPipeline": 600,
diff --git a/locations/spiders/aldi_sud_us.py b/locations/spiders/aldi_sud_us.py
--- a/locations/spiders/aldi_sud_us.py
+++ b/locations/spiders/aldi_sud_us.py
@@ -6,7 +6,7 @@
 
 class AldiSudUSSpider(SitemapSpider, StructuredDataSpider):
     name = "aldi_sud_us"
-    item_attributes = {"brand": "ALDI", "brand_wikidata": "Q41171672"}
+    item_attributes = {"brand": "ALDI", "brand_wikidata": "Q41171672", "country": "US"}
     allowed_domains = ["stores.aldi.us"]
     sitemap_urls = ["https://stores.aldi.us/robots.txt"]
     sitemap_rules = [


Test patch: