Repository: alltheplaces/alltheplaces

Issue:
Ralph Lauren
https://www.ralphlauren.com/Stores-ShowCountries


Discussion:
Please add more information about the issue.
@YorozuyaDev Hello, thanks for reaching out. I've labeled issues in the repository that are good options for creating your first spider. The ultimate goal is to scrape the URL linked above (or a better one if you find it) for [location information](https://github.com/alltheplaces/alltheplaces/blob/master/DATA_FORMAT.md) regarding the business. The project is built using [scrapy](https://docs.scrapy.org/en/latest/), so there is more documentation there the can also assist you as well. Feel free to comb through the spiders in the locations folder as well. If you find an issue that you think looks interesting, go ahead and assign it to yourself and feel free to reach out if you have more questions.
@thekuiperbelt I'd like to take this one too!
@moohco Awesome! Please assign it to yourself. Happy crawling, I hope you enjoy it. 🕷️ 

Resolving patch:
diff --git a/locations/spiders/ralph_lauren.py b/locations/spiders/ralph_lauren.py
--- a/locations/spiders/ralph_lauren.py
+++ b/locations/spiders/ralph_lauren.py
@@ -1,44 +1,72 @@
 # -*- coding: utf-8 -*-
 import scrapy
 import json
+import base64
 from locations.items import GeojsonPointItem
 
-class LanghamHotel(scrapy.Spider):
-    name = "langham"
-    allowed_domains = ["www.langhamhotels.com","www.cordishotels.com", "www.langhamresidences.com"]
+class RalphLauren(scrapy.Spider):
+    name = "ralph_lauren"
+    allowed_domains = ["www.ralphlauren.com"]
     start_urls = (
-        'https://www.langhamhotels.com/en/locations/',
+        'https://www.ralphlauren.com/Stores-ShowCountries',
     )
 
     def parse(self, response):
-        #gather URLs of all Langham Hotel locations
-        all_locations = response.xpath('//div[@class="listing-item locations-item"]/a/@href').extract()
+        #gather URLs for all countries
+        countries = response.xpath('//a[@class="store-directory-countrylink"]/@href').extract()
 
-        for locations in all_locations:
-            yield scrapy.Request(response.urljoin(locations),callback=self.parse_locations)
+        for country in countries:
+            yield scrapy.Request(response.urljoin(country),callback=self.parse_city)
 
-    def parse_locations(self,response):
-        data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
+    def parse_city(self, response):
+        #all URLs for cities in the countries
+        cities = response.xpath('//a[@class="store-directory-citylink"]/@href').extract()
+
+        for city in cities:
+            yield scrapy.Request(response.urljoin(city), callback=self.parse_stores)
+
+    def parse_stores(self, response):
+        # if a city has more than 1 store
+        if len(response.xpath('//span[@class="store-listing-name"]/a/@href').extract()) > 0:
+            stores = response.xpath('//span[@class="store-listing-name"]/a/@href').extract()
+
+            for store in stores:
+                yield scrapy.Request(response.urljoin(store), callback=self.parse_locations)
+        else:
+            stores = response.xpath('//a[@class="store-directory-citylink"]/@href').extract()
+
+            for store in stores:
+                yield scrapy.Request(response.urljoin(store), callback=self.parse_locations)
+
+    def parse_locations(self, response):
+        data = response.xpath('//div[@class="storeJSON hide"]/@data-storejson').extract_first()
+
+        hours = response.xpath('//tr[@class="store-hourrow"]//td//text()').getall()
+        opening_hours = []
+
+        for i in hours:
+            opening_hours.append(i.strip())
 
         if data and self.isValidJson(data):
             try:
-                data = json.loads(data)
-
-                # preparation to extract data in the next step, return empty if there is no geo/address in the json
-                # otherwise throws error while setting properties
-                geo_data = data.get("geo", {})
-                address_data = data.get("address", {})
+                data = json.loads(data)[0]
+                name = data.get("name", None)
+                name = base64.b64decode(name).decode('utf-8')
+                address = data.get("address1", None)
+                address = base64.b64decode(address).decode('utf-8')
 
-                # all properties set with data.get to set proper value or None if empty/non-existent
                 properties = {
-                    'ref': response.url,
-                    'name': data.get("name", None),
-                    'lat': geo_data.get("latitude", None),
-                    'lon': geo_data.get("longitude", None),
-                    'phone': data.get("telephone", None),
-                    'addr_full': address_data.get("streetAddress", None),
-                    'country': address_data.get("addressCountry", None),
-                    'postcode': address_data.get("postalCode", None),
+                    'ref': data.get("id", None),
+                    'name': name,
+                    'lat': data.get("latitude", None),
+                    'lon': data.get("longitude", None),
+                    'phone': data.get("phone", None),
+                    'addr_full': address,
+                    'state': data.get("stateCode", None),
+                    'city': data.get("city", None),
+                    'country': data.get("countryCode", None),
+                    'postcode': data.get("postalCode", None),
+                    'opening_hours': opening_hours,
                 }
 
                 yield GeojsonPointItem(**properties)
@@ -46,15 +74,11 @@ def parse_locations(self,response):
             except Exception as e:
                 self.logger.warn("----------------- Error -----------------: {}".format(e))
 
-        else:
-            # some of the websites don't provide location information with application/ld+json
-            # or return invalid JSON
-            # the only other option is to crawl the footers in that case, which seems messy to me
-            pass
-
     def isValidJson(self,jsondata):
         try:
             json.loads(jsondata)
         except Exception as e:
+            self.logger.warn("----------------- Error -----------------: {}".format(e))
             return False
         return True
+    
\ No newline at end of file


Test patch: