Repository: alltheplaces/alltheplaces

Issue:
Co-Op Credit Unions
(via a shared branching site, possibly out of scope? http://co-opcreditunions.org/locator/ )


Discussion:
Going to clear the assignee on this so others can work on it. If you're still working on it, please re-assign to yourself and push up what you've got so far.
Is there any new issue now?
All of issues are self assigned.

@iandees 
I can't find opened issues.
Can you open new issues for me now?
There are several tickets available: https://github.com/alltheplaces/alltheplaces/issues?utf8=✓&q=no%3Aassignee+is%3Aissue+is%3Aopen+-label%3Aquestion
@iandees 
The opened issues have some problems, so can you assign me other available issues?
What problems do you see? If you're running into problems please document them on the issue so others don't waste their time on it.

For now, please feel free to write spiders for places that aren't already listed. It's especially useful to include places that are local to you so we can include a wider variety of places.
@iandees 
ok
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.

Resolving patch:
diff --git a/locations/spiders/creditunion.py b/locations/spiders/creditunion.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/creditunion.py
@@ -0,0 +1,101 @@
+import scrapy
+import re
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+STATES = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA",
+          "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
+          "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
+          "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
+          "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"]
+
+
+class CreditUnionSpider(scrapy.Spider):
+
+    name = "creditunion"
+    download_delay = 0.5
+    allowed_domains = [
+        "co-opcreditunions.org"
+    ]
+
+    def start_requests(self):
+        base_url = 'https://co-opcreditunions.org/locator/search-results/?loctype=S&state={state}&statewide=yes&country=&Submit=Search&lp=1'
+
+        for state in STATES:
+            url = base_url.format(state=state)
+            yield scrapy.Request(url)
+
+    def parse_hours(self, days):
+        opening_hours = OpeningHours()
+        for day in days:
+            day = day.strip('\n ')
+            if not day:
+                continue
+            parts = day.split(':')
+            weekday = parts[0]
+            hours = ":".join(parts[1:])
+            try:
+                open, close = hours.split('-')
+                if (not open.strip()) or (open.strip() == 'Closed'):
+                    continue
+                if close.strip() in ('24:00', '23:59'):  # two oddball banks
+                    continue
+                opening_hours.add_range(day=weekday[:2],
+                                        open_time="{} AM".format(open.strip()),
+                                        close_time="{} PM".format(close.strip()),
+                                        time_format="%I:%M %p")
+            except:
+                continue
+
+        return opening_hours.as_opening_hours()
+
+
+    def parse_bank(self, bank):
+
+        address = bank.xpath('.//address/text()').extract()
+        try:
+            phone = address[3].strip()
+        except IndexError:
+            phone = ''
+
+        href = bank.xpath('.//div[@class="location-results__share"]/div/a/@href').extract_first()
+        lat, lon = re.search(r'lat=(.*?)&lng=(.*?)&', href).groups()
+        name = bank.xpath('.//h3/text()').extract_first()
+        ref = name + "-" + "-".join(address[0].split(' '))
+
+        properties = {
+            'name': name,
+            'addr_full': address[0].strip(),
+            'city': address[1].split(',')[0].strip(),
+            'state': address[1].split(',')[1].strip(),
+            'postcode': address[2].strip(),
+            'phone': phone,
+            'ref': ref,
+            'website': bank.xpath('./div[1]/a/@href').extract_first(),
+            'lat':  float(lat),
+            'lon':  float(lon),
+            'extras': {
+                'brand': name
+            }
+        }
+        hours = self.parse_hours(bank.xpath('.//div[contains(@class, "location-results__hours")]/p/text()').extract())
+        if hours:
+            properties['opening_hours'] = hours
+        return properties
+
+    def parse(self, response):
+        page_num = int(re.search(r'lp=(\d+)$', response.url).groups()[0])
+        pages = response.xpath('//select[@class="locator-pager-select"]/option/text()').extract()
+        if pages:
+            max_page = max([int(p) for p in pages])
+        else:
+            max_page = 1
+
+        banks = response.xpath('//div[@class="location-results"]/div')
+        for bank in banks:
+            properties = self.parse_bank(bank)
+            yield GeojsonPointItem(**properties)
+
+        if page_num < max_page:
+            yield scrapy.Request(response.url.replace('lp={}'.format(page_num), 'lp={}'.format(page_num+1)))


Test patch: