Repository: alltheplaces/alltheplaces

Issue:
Deichmann
https://stores.deichmann.com/index.html

Planning to put more Germany/Europe on the map :)


Discussion:

Resolving patch:
diff --git a/locations/spiders/deichmann.py b/locations/spiders/deichmann.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/deichmann.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import base64
+from locations.items import GeojsonPointItem
+
+class Deichmann(scrapy.Spider):
+    name = "deichmann"
+    allowed_domains = ["stores.deichmann.com","stores.dosenbach.ch"]
+    start_urls = (
+        'https://stores.deichmann.com/index.html',
+    )
+
+    def parse(self, response):
+        # base url of all countries
+        countries = response.xpath('//ul[@class="countrydropdown"]/li/a/@href').extract()
+
+        for country in countries:
+            yield scrapy.Request(country, callback=self.parse_country)
+
+    def parse_country(self, response):
+        # parse country per region
+        cities = response.xpath('//a[@class="region-list__link"]/@href').extract()
+
+        for city in cities:
+            yield scrapy.Request(response.urljoin(city), callback=self.parse_city)
+
+    def parse_city(self, response):
+        # parse cities in region
+        stores = response.xpath('//a[@class="button-link"]/@href').extract()
+
+        for store in stores:
+            yield scrapy.Request(response.urljoin(store), callback=self.parse_locations)
+
+    def parse_locations(self, response):
+        # get json which provides all data
+        data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
+
+        try:
+            if data:
+                data = json.loads(data)
+
+                geo_data = data.get("geo", {})
+                address_data = data.get("address", {})
+                contact_data = data.get("ContactPoint", {})
+
+                properties = {
+                    'ref': data.get("@id", None),
+                    'name': data.get("name", None),
+                    'lat': geo_data.get("latitude", None),
+                    'lon': geo_data.get("longitude", None),
+                    'phone': contact_data[0].get("telephone", None),
+                    'addr_full': address_data.get("streetAddress", None),
+                    'country': address_data.get("addressCountry", None),
+                    'postcode': address_data.get("postalCode", None),
+                    'city': address_data.get("addressLocality", None),
+                    'opening_hours': data.get("openingHours", None),
+                }
+
+                yield GeojsonPointItem(**properties)
+        except Exception as e:
+            self.logger.warn("----------------- Error -----------------: {}".format(e))
+            pass
diff --git a/locations/spiders/langham_hotels.py b/locations/spiders/langham_hotels.py
--- a/locations/spiders/langham_hotels.py
+++ b/locations/spiders/langham_hotels.py
@@ -20,7 +20,7 @@ def parse(self, response):
     def parse_locations(self,response):
         data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
 
-        if data and self.isValidJson(data):
+        if data:
             try:
                 data = json.loads(data)
 
@@ -45,6 +45,7 @@ def parse_locations(self,response):
 
             except Exception as e:
                 self.logger.warn("----------------- Error -----------------: {}".format(e))
+                pass
 
         else:
             # some of the websites don't provide location information with application/ld+json
@@ -60,11 +61,3 @@ def parse_locations(self,response):
             # https://www.langhamhotels.com/en/the-langham/haikou/
             # https://www.langhamresidences.com/en/the-langham/nymphenburg/ (no location information)
             pass
-
-    def isValidJson(self,jsondata):
-        try:
-            json.loads(jsondata)
-        except Exception as e:
-            self.logger.warn("----------------- Error -----------------: {}".format(e))
-            return False
-        return True


Test patch: