Repository: alltheplaces/alltheplaces

Issue:
Rent-A-Center
Index of all states near the bottom of this page: https://locations.rentacenter.com/


Discussion:
Search with ZIP, dynamic loading pages, requests with security tokens.
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.

Resolving patch:
diff --git a/locations/spiders/rentacenter.py b/locations/spiders/rentacenter.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/rentacenter.py
@@ -0,0 +1,90 @@
+import json
+import scrapy
+import re
+from scrapy.selector import Selector
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'Monday': 'Mo',
+    'Tuesday': 'Tu',
+    'Wednesday': 'We',
+    'Thursday': 'Th',
+    'Friday': 'Fr',
+    'Saturday': 'Sa',
+    'Sunday': 'Su'
+}
+
+
+class RentACenterSpider(scrapy.Spider):
+    name = "rentacenter"
+    allowed_domains = ["rentacenter.com"]
+
+    start_urls = [
+        "https://locations.rentacenter.com/sitemap.xml",
+    ]
+
+    def parse_hours(self, hours):
+        opening_hours = OpeningHours()
+
+        for hour in hours:
+            opening_hours.add_range(day=DAY_MAPPING[hour["dayOfWeek"].replace('http://schema.org/', '')],
+                                    open_time=hour["opens"],
+                                    close_time=hour["closes"],
+                                    time_format='%H:%M:%S')
+
+        return opening_hours.as_opening_hours()
+
+    def parse_location(self, response):
+        data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
+        data = json.loads(data)
+
+        ref = data.get("branchCode")
+        if not ref:
+            return  # not a store page
+
+        properties = {
+            'addr_full': data["address"]["streetAddress"],
+            'phone': data["telephone"],
+            'city': data["address"]["addressLocality"],
+            'state': data["address"]["addressRegion"],
+            'postcode': data["address"]["postalCode"],
+            'country': 'US',
+            'ref': ref,
+            'website': response.url,
+            'lat': float(data["geo"]["latitude"]),
+            'lon': float(data["geo"]["longitude"]),
+            'name': data["name"]
+        }
+
+        hours = self.parse_hours(data.get("openingHoursSpecification", []))
+        if hours:
+            properties["opening_hours"] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse_state_sitemap(self, response):
+        xml = Selector(response)
+        xml.remove_namespaces()
+
+        urls = xml.xpath('//loc/text()').extract()
+        urls = [url.strip() for url in urls]
+
+        # individual store pages are listed at top, then a state page, then bunch of other non-store pages
+        # find the index position of the state page and then only parse urls before that
+        i = urls.index(re.search(r'^(https://locations.rentacenter.com/.+?)/.*$', urls[0]).groups()[0] + '/')
+        for url in urls[:i]:
+            yield scrapy.Request(url, callback=self.parse_location)
+
+    def parse(self, response):
+        xml = Selector(response)
+        xml.remove_namespaces()
+
+        urls = xml.xpath('//loc/text()').extract()
+        urls = [url.strip() for url in urls]
+
+        for url in urls:
+            yield scrapy.Request(url, callback=self.parse_state_sitemap)
+


Test patch: