Repository: biglocalnews/court-scraper (language: python)

Issue:
Refactor OdysseySite to make login option for San Mateo County
San Mateo County doesn't require a login for the Odyssey platform. Refactor Odyssey site class to make login optional.

- [x] Update Odyssey Site `__init__`
   - [x] to remove user/pass
   - [x] to prepare driver (currently handled in `login`)
   - [x] Pass `headless` as argument
- [x] Add user/pass as keyword arguments to `Site.login`
- [x] Remove `headless` argument from `login` and `search` method signatures
- [x] Update `Runner` to only call login when password creds are present
Update OdysseySite.search to optionally skip fetching of case detail page
Update `OdysseySite.search` to allow option to skip fetching of case detail information. This is useful for simpler use case when we only care about top-level case metadata listed on search result pages, for example when we're compiling a basic index of available cases as part of "case discovery".


Discussion / hints:

Fix patch:
diff --git a/court_scraper/__init__.py b/court_scraper/__init__.py
index e69de29..b794fd4 100644
--- a/court_scraper/__init__.py
+++ b/court_scraper/__init__.py
@@ -0,0 +1,1 @@
+__version__ = '0.1.0'
diff --git a/court_scraper/base/selenium_site.py b/court_scraper/base/selenium_site.py
index fffab8b..d6bf1bc 100644
--- a/court_scraper/base/selenium_site.py
+++ b/court_scraper/base/selenium_site.py
@@ -15,23 +15,23 @@ def _init_chrome_driver(self, headless=True):
 
     def _build_chrome_options(self, headless=True, random_user=False):
         chrome_options = Options()
+        chrome_options.add_argument("--disable-notifications")
+        chrome_options.add_argument('--verbose')
+        chrome_options.add_argument("--window-size=1920x1080")
+        chrome_options.add_argument('--no-sandbox')
+        chrome_options.add_experimental_option("prefs", {
+            "download.default_directory": self.download_dir,
+            "download.prompt_for_download": False,
+            "download.directory_upgrade": True,
+            "safebrowsing_for_trusted_sources_enabled": False,
+            "safebrowsing.enabled": False
+        })
+        chrome_options.add_argument('--disable-gpu')
+        chrome_options.add_argument('--disable-software-rasterizer')
         if headless:
             chrome_options.add_argument("--headless")
         if random_user:
-            chrome_options.add_argument("--window-size=1920x1080")
-            chrome_options.add_argument("--disable-notifications")
-            chrome_options.add_argument('--no-sandbox')
-            chrome_options.add_argument('--verbose')
-            chrome_options.add_experimental_option("prefs", {
-                    "download.default_directory": self.download_dir,
-                    "download.prompt_for_download": False,
-                    "download.directory_upgrade": True,
-                    "safebrowsing_for_trusted_sources_enabled": False,
-                    "safebrowsing.enabled": False
-            })
-            chrome_options.add_argument('--disable-gpu')
-            chrome_options.add_argument('--disable-software-rasterizer')
             ua = UserAgent(family='chrome')
             randomua = ua.random()
             chrome_options.add_argument(f'user-agent={randomua}')
-        return chrome_options
\ No newline at end of file
+        return chrome_options
diff --git a/court_scraper/case_info.py b/court_scraper/case_info.py
index 6f46145..924ab62 100644
--- a/court_scraper/case_info.py
+++ b/court_scraper/case_info.py
@@ -39,8 +39,15 @@ def standard_data(self):
         Return data dict of fields that map
         to Case table columns.
         """
-        return {
+        data = {
             'place_id': self.place_id,
             'number': self.number,
-            'status': self.status,
         }
+        # Strongly encouraged, not always available
+        for attr in ['filing_date', 'status']:
+            try:
+                data[attr] = getattr(self, attr)
+            except AttributeError:
+                data[attr] = None
+        return data
+
diff --git a/court_scraper/platforms/odyssey_site/pages/search_results.py b/court_scraper/platforms/odyssey_site/pages/search_results.py
index b2d8c6a..4801830 100644
--- a/court_scraper/platforms/odyssey_site/pages/search_results.py
+++ b/court_scraper/platforms/odyssey_site/pages/search_results.py
@@ -15,6 +15,7 @@ class SearchResultsPageLocators:
     RESULTS_DIV = (By.CSS_SELECTOR, '#SmartSearchResults')
     NO_RESULTS_MSG = (By.XPATH, '//*[@id="ui-tabs-1"]/div/p')
     CASE_DETAIL_LINK = (By.CSS_SELECTOR, 'a.caseLink')
+    RESULT_HEADERS = (By.CSS_SELECTOR, 'th.k-header')
     # Get table rows that are the grandparent of case links;
     # they contain all case metadata in search results.
     RESULT_ROWS = (By.XPATH, "//a[@class='caseLink']/../..")
@@ -34,6 +35,27 @@ class SearchResultsPageLocators:
 
 
 # Elements
+class ResultHeaders:
+    """Gets field names in search results"""
+
+    locator = SearchResultsPageLocators.RESULT_HEADERS
+
+    def __init__(self, driver):
+        self.driver = driver
+
+    @property
+    def values(self):
+        driver = self.driver
+        WebDriverWait(driver, 100).until(
+            lambda driver: driver.find_elements(*self.locator)
+        )
+        return [
+            el.text.strip()
+            for el in driver.find_elements(*self.locator)
+            if el.text.strip()
+        ]
+
+
 class SearchResults:
 
     locator = SearchResultsPageLocators.RESULT_ROWS
@@ -45,19 +67,21 @@ def __get__(self, obj, owner):
             lambda driver: driver.find_elements(*self.locator)
         )
         elements = driver.find_elements(*self.locator)
-        case_rows = self._prep_case_rows(elements)
+        headers = ResultHeaders(driver).values
+        case_rows = self._prep_case_rows(headers, elements)
         return case_rows
 
-    def _prep_case_rows(self, elements):
+    def _prep_case_rows(self, headers, elements):
         case_rows = []
         for el in elements:
-            case_rows.append(ResultRow(el))
+            case_rows.append(ResultRow(headers, el))
         return case_rows
 
 
 class ResultRow:
 
-    def __init__(self, row_element):
+    def __init__(self, headers, row_element):
+        self.headers = headers
         self.el = row_element
 
     @property
@@ -65,21 +89,16 @@ def metadata(self):
         case_detail_url = self.el.find_element(
                 *SearchResultsPageLocators.CASE_DETAIL_LINK
             ).get_attribute('data-url')
-        case_num, style_def, file_date, status, party_name = [
-            field.strip() for field in self.inner_text.split('\t')
-        ]
-        return {
-            'case_num': case_num,
-            'style_defendant': style_def,
-            'file_date': file_date,
-            'status': status,
-            'party_name': party_name,
-            'case_detail_url': case_detail_url,
-        }
+        data = dict(zip(self.headers, self.values))
+        data['case_detail_url'] = case_detail_url
+        return data
 
     @property
-    def inner_text(self):
-        return self.el.get_attribute('innerText').strip('\t')
+    def values(self):
+        return [
+            el.text.strip() for el in self.el.find_elements_by_xpath('child::*')
+            if el.text.strip()
+        ]
 
     @property
     def detail_page_link(self):
diff --git a/court_scraper/platforms/odyssey_site/site.py b/court_scraper/platforms/odyssey_site/site.py
index 8f535f8..b9bd1cf 100644
--- a/court_scraper/platforms/odyssey_site/site.py
+++ b/court_scraper/platforms/odyssey_site/site.py
@@ -10,15 +10,15 @@
 
 class OdysseySite(SeleniumSite):
 
-    def __init__(self, url, username, password, download_dir, timeout=60):
+    def __init__(self, url, download_dir, timeout=60, headless=True):
         self.site_url = url
-        self.username = username
-        self.password = password
         self.download_dir = download_dir
         self.timeout = timeout
-
-    def login(self, headless=True):
         self.driver = self._init_chrome_driver(headless=headless)
+
+    def login(self, username, password):
+        self.username = username
+        self.password = password
         login_page = LoginPage(
             self.driver,
             self.site_url,
@@ -28,7 +28,7 @@ def login(self, headless=True):
         login_page.go_to()
         login_page.login()
 
-    def search(self, search_terms=[], headless=True):
+    def search(self, search_terms=[], case_details=True):
         portal_page = PortalPage(self.driver)
         portal_page.go_to_smart_search()
         results = []
@@ -42,12 +42,13 @@ def search(self, search_terms=[], headless=True):
                 if results_page.results_found():
                     for case_row in results_page.results:
                         row_data = case_row.metadata
-                        case_row.detail_page_link.click()
-                        detail_page = CaseDetailPage(self.driver)
-                        row_data['page_source'] = detail_page.page_source
+                        if case_details:
+                            case_row.detail_page_link.click()
+                            detail_page = CaseDetailPage(self.driver)
+                            row_data['page_source'] = detail_page.page_source
+                            results_page.back_to_search_results()
                         ci = CaseInfoKls(row_data)
                         results.append(ci)
-                        results_page.back_to_search_results()
                 results_page.back_to_smart_search_tab()
             return results
         finally:
@@ -55,11 +56,11 @@ def search(self, search_terms=[], headless=True):
 
     def _get_case_info_mapped_class(self):
         mapping = {
-            'case_num': 'number',
-            'file_date': 'filing_date',
+            'Case Number': 'number',
+            'Style / Defendant': 'defendant',
+            'File Date': 'filing_date',
+            'Party Name': 'party',
+            'Status': 'status',
         }
         CaseInfo._map = mapping
         return CaseInfo
-
-    
-
diff --git a/court_scraper/runner.py b/court_scraper/runner.py
index 0322dc3..59ed0ae 100644
--- a/court_scraper/runner.py
+++ b/court_scraper/runner.py
@@ -50,18 +50,17 @@ def search(self, search_terms=[], headless=True):
         url = self.site_meta['home_url']
         username, password = self._get_login_creds()
         pos_args = [url]
-        if username and password:
-            pos_args.extend([username, password])
         site = SiteKls(
             *pos_args,
             self.cache_dir,
+            headless=headless
         )
         if username and password:
-            site.login(headless=headless)
+            site.login(username, password)
         logger.info(
             "Executing search for {}".format(self.place_id)
         )
-        data = site.search(search_terms=search_terms, headless=headless)
+        data = site.search(search_terms=search_terms)
         return data
 
 
@@ -96,6 +95,7 @@ def parse_html_pages(self, html_pages):
         Base parser provides functionality for opening html and saving json;
 
         """
+        pass
 
     def _get_site_class(self):
         # Site types for one-off scrapers should live in the scrapers
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..31af119
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,15 @@
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+
+[bumpversion:file:setup.py]
+
+[bumpversion:file:court_scraper/__init__.py]
+
+[flake8]
+max-line-length = 110
+exclude = docs,
+	__pycache__,
+	.git
+


Test patch:
diff --git a/tests/platforms/test_odyssey_site.py b/tests/platforms/test_odyssey_site.py
index 50a28e7..5f16dcd 100644
--- a/tests/platforms/test_odyssey_site.py
+++ b/tests/platforms/test_odyssey_site.py
@@ -8,16 +8,12 @@
 @patch('court_scraper.platforms.odyssey_site.site.LoginPage')
 @patch('court_scraper.base.selenium_site.webdriver')
 def test_login(webdriver_mock, login_page_mock):
-    site = OdysseySite(
-        'http://somesite.com',
-        'user',
-        'pass',
-        '/some_path/'
-    )
-    site.login()
-    # Login initializes chrome driver
+    site = OdysseySite('http://somesite.com', '/tmp/some_path/')
+    # Web driver instantiated during Site class initialization
     assert webdriver_mock.Chrome.called
-    # and calls login
+    # Login requires password
+    site.login('user', 'pass')
+    # login method goes to login page and submits user creds
     expected_calls = [
         call().go_to(),
         call().login()
@@ -47,7 +43,34 @@ def test_search(test_input, headless, live_configs, court_scraper_dir):
     username = auth['username']
     password = auth['password']
     url, case_ids = test_input
-    site = OdysseySite(url, username, password, download_dir=court_scraper_dir)
-    site.login(headless=headless)
-    results = site.search(search_terms=case_ids, headless=False)
+    site = OdysseySite(url, download_dir=court_scraper_dir, headless=headless)
+    site.login(username, password)
+    results = site.search(search_terms=case_ids)
     assert len(results) == 1
+    # Scrapes Case Detail page HTML by default
+    assert 'page_source' in results[0].data.keys()
+
+
+@pytest.mark.parametrize(
+    "test_input",
+    [
+        (
+            "https://ody.dekalbcountyga.gov/portal/Home/Dashboard/29",
+            ["19d89169"]
+        ),
+    ]
+)
+@pytest.mark.slow
+@pytest.mark.webtest
+def test_skip_case_details(test_input, headless, live_configs, court_scraper_dir):
+    "should support option to skip scraping of case details"
+    auth = live_configs['ga_dekalb']
+    username = auth['username']
+    password = auth['password']
+    url, case_ids = test_input
+    site = OdysseySite(url, download_dir=court_scraper_dir, headless=headless)
+    site.login(username, password)
+    results = site.search(search_terms=case_ids, case_details=False)
+    # Should *NOT* have case detail HTML stored on return object
+    assert len(results) == 1
+    assert 'page_source' not in results[0].data.keys()
diff --git a/tests/test_case_info.py b/tests/test_case_info.py
index f9f4bb3..7698d40 100644
--- a/tests/test_case_info.py
+++ b/tests/test_case_info.py
@@ -27,7 +27,9 @@ def test_standardized_data():
         'place_id': 'ga_dekalb',
         'number': '1',
         'status': 'Open',
+        'filing_date': None,
     }
     CaseInfo._map = mapping
     ci = CaseInfo(data)
     assert ci.standard_data == expected
+
diff --git a/tests/test_runner.py b/tests/test_runner.py
index 1e712dc..b65f36f 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -26,16 +26,19 @@ def test_site_calls(court_scraper_dir, config_path):
         r.search(search_terms=['foo'])
         username = 'user1@example.com'
         password = 'SECRETPASS'
-        expected_args = [
+        expected_args = (
             'https://ody.dekalbcountyga.gov/portal/Home/Dashboard/29',
-            username,
-            password,
-            court_scraper_dir
-        ]
-        site_class.assert_called_once_with(*expected_args)
+            court_scraper_dir,
+        )
+        expected_kwargs = {'headless': True}
+        # Get the args and kwargs (2nd and 3rd items) from the 
+        # first call which is Site instantiation
+        args, kwargs = site_class.mock_calls[0][1:]
+        assert args == expected_args
+        assert kwargs == expected_kwargs
         login_call, search_call = site_class.mock_calls[1:3]
-        assert login_call == call().login(headless=True)
-        assert search_call == call().search(search_terms=['foo'],headless=True)
+        assert login_call == call().login(username, password)
+        assert search_call == call().search(search_terms=['foo'])
 
 
 @pytest.mark.usefixtures('create_scraper_dir', 'create_config')