Repository: alltheplaces/alltheplaces

Issue:
lloyds_bank_gb spider doesn't capture addresses
I'm not sure how easy / feasible this would be to fix, but the lloyds_bank_gb spider doesn't currently pick up the postcode or address for each branch. I don't know the details of what the spidering framework can detect by default, but the address data does seem to be included in the HTML in some sort of structured way.


Discussion:
```python
'address': [{'@type': 'PostalAddress'},
             {'@type': 'PostalAddress',
              'addressLocality': 'Aldershot',
              'streetAddress': '115 Victoria Road',
              'postalCode': 'GU11 1JQ',
              'addressCountry': 'GB'}]
```

The issue here is that they actually give us 2 addresses. We currently just take the first (and I think we reversed the order recently). We could easily use the last in this spider.

```python
    def pre_process_data(self, ld_data, **kwargs):
        ld_data["address"] = ld_data["address"][-1]
```

Or maybe we should merge multiple addresses in shared code... I don't know if that would backfire in any other spider though.
When we see a `schema:PostalAddress` that has no other properties (or a `schema:GeoCoordinates` or really anything) it's probably safe to ignore it entirely.

Could do this here:

https://github.com/alltheplaces/alltheplaces/blob/51b0d65c4d6bd37ff4b8c4765dc421d4d0bcde72/locations/microdata_parser.py#L257

e.g.

```python
if not item["properties"]:
    return None
```
I think the same issue is also present in the halifax_gb and bank_of_scotland_gb spiders. So it might be worth checking if the solutions proposed for Lloyds also work there.
I'll test that fix with those spiders as tests and let everyone know

Resolving patch:
diff --git a/locations/microdata_parser.py b/locations/microdata_parser.py
--- a/locations/microdata_parser.py
+++ b/locations/microdata_parser.py
@@ -254,8 +254,16 @@ def convert_item(item):
     # Properties is a list; if its length is 1 then flatten, else don't.
     if itemid := item.get("id"):
         ld["@id"] = itemid
+    if len(item["properties"].items()) == 0:
+        # Guard against goofy use of microdata such as:
+        # <a itemscope itemprop=address itemtype=PostalAddress>...</a>
+        # Which produce an item with no properties and make it difficult to
+        # parse the correct item later down. See test_multiple_addresses
+        return
     for k, v in item["properties"].items():
-        ld[k] = [convert_item(val) if isinstance(val, dict) else val for val in v]
+        ld[k] = filter(
+            None, [convert_item(val) if isinstance(val, dict) else val for val in v]
+        )
         ld[k] = remove_duplicates(ld[k])
         if len(ld[k]) == 1:
             ld[k] = ld[k][0]
diff --git a/locations/spiders/halifax_gb.py b/locations/spiders/halifax_gb.py
--- a/locations/spiders/halifax_gb.py
+++ b/locations/spiders/halifax_gb.py
@@ -1,16 +1,20 @@
 from scrapy.spiders import SitemapSpider
 
+from locations.categories import Categories
 from locations.structured_data_spider import StructuredDataSpider
 
 
 class HalifaxGB(SitemapSpider, StructuredDataSpider):
     name = "halifax_gb"
-    item_attributes = {"brand": "Halifax", "brand_wikidata": "Q3310164"}
+    item_attributes = {
+        "brand": "Halifax",
+        "brand_wikidata": "Q3310164",
+        "extras": Categories.BANK.value,
+    }
     sitemap_urls = ["https://branches.halifax.co.uk/sitemap.xml"]
     sitemap_rules = [
         (r"https:\/\/branches\.halifax\.co\.uk\/[-'\w]+\/[-'\/\w]+$", "parse_sd")
     ]
-    wanted_types = ["BankOrCreditUnion"]
 
     def sitemap_filter(self, entries):
         for entry in entries:


Test patch:
diff --git a/tests/test_microdata_parser.py b/tests/test_microdata_parser.py
--- a/tests/test_microdata_parser.py
+++ b/tests/test_microdata_parser.py
@@ -7,7 +7,7 @@ def test_itemref():
     src = """
 <main itemscope itemtype="http://schema.org/GroceryStore">
     <article itemref="schema-location" itemprop="event" itemscope itemtype="http://schema.org/Event">
-        <div id="schema-location" itemprop="location" itemscope itemtype="http://schema.org/Place"></div>
+        <div id="schema-location" itemprop="location" itemscope itemtype="http://schema.org/Place"><span itemprop="name">Test</span></div>
     </article>
 </main>
     """
@@ -25,7 +25,7 @@ def test_itemref():
                                 "location": [
                                     {
                                         "type": ["http://schema.org/Place"],
-                                        "properties": {},
+                                        "properties": {"name": ["Test"]},
                                     }
                                 ]
                             },
@@ -39,5 +39,77 @@ def test_itemref():
     assert ld == {
         "@context": "https://schema.org",
         "@type": "GroceryStore",
-        "event": {"@type": "Event", "location": {"@type": "Place"}},
+        "event": {"@type": "Event", "location": {"@type": "Place", "name": "Test"}},
+    }
+
+
+# Example from https://branches.lloydsbank.com/aldershot/115-victoria-road where there are multiple address objects,
+# but one is empty
+def test_multiple_addresses():
+    src = """
+<main itemscope itemtype="http://schema.org/BankOrCreditUnion" itemid="https://branches.lloydsbank.com/#15682237">
+    <span itemprop="name"><span class="LocationName-brand">Lloyds Bank</span> <span class="LocationName-geo">Aldershot</span></span>
+    <a href="" itemscope itemtype="http://schema.org/PostalAddress" itemprop="address">
+        <div class="Core-mobileAddressContainer">
+            <span class="Core-addressRow">115 Victoria Road</span>
+            <span class="Core-addressRow">Aldershot, GU11 1JQ</span>
+        </div>
+    </a>
+    <address itemscope itemtype="http://schema.org/PostalAddress" itemprop="address" data-country="GB">
+        <meta itemprop="addressLocality" content="Aldershot">
+        <meta itemprop="streetAddress" content="115 Victoria Road">
+        <div class="c-AddressRow">
+            <span class="c-address-street-1">115 Victoria Road</span>
+        </div>
+        <div class="c-AddressRow">
+            <span class="c-address-city">Aldershot</span>
+            <span class="c-address-postal-code" itemprop="postalCode">GU11 1JQ</span>
+        </div>
+        <div class="c-AddressRow">
+            <abbr title="United Kingdom" aria-label="United Kingdom" class="c-address-country-name c-address-country-gb" itemprop="addressCountry">GB</abbr>
+        </div>
+    </address>
+</main>
+    """
+    doc = Selector(text=src)
+    items = MicrodataParser.extract_microdata(doc)
+    assert items == {
+        "items": [
+            {
+                "id": "https://branches.lloydsbank.com/#15682237",
+                "type": ["http://schema.org/BankOrCreditUnion"],
+                "properties": {
+                    "name": ["Lloyds Bank Aldershot"],
+                    "address": [
+                        {
+                            "type": ["http://schema.org/PostalAddress"],
+                            "properties": {},
+                        },
+                        {
+                            "type": ["http://schema.org/PostalAddress"],
+                            "properties": {
+                                "addressLocality": ["Aldershot"],
+                                "streetAddress": ["115 Victoria Road"],
+                                "postalCode": ["GU11 1JQ"],
+                                "addressCountry": ["GB"],
+                            },
+                        },
+                    ],
+                },
+            }
+        ]
+    }
+    ld = MicrodataParser.convert_to_graph(items)
+    assert ld == {
+        "@context": "https://schema.org",
+        "@id": "https://branches.lloydsbank.com/#15682237",
+        "@type": "BankOrCreditUnion",
+        "name": "Lloyds Bank Aldershot",
+        "address": {
+            "@type": "PostalAddress",
+            "addressLocality": "Aldershot",
+            "streetAddress": "115 Victoria Road",
+            "postalCode": "GU11 1JQ",
+            "addressCountry": "GB",
+        },
     }