Repository: sec-edgar/sec-edgar
Pull request #106 (2020-05-16 19:08:37)

Extend testing and add new CI for GitHub Actions
- [x] tests added / passed
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [ ] added entry to docs/whatsnew latest version


Patch:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..c07c361
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,35 @@
+name: Build Docs
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout 🛎️
+        # If you're using actions/checkout@v2 you must
+        # set persist-credentials to false in most cases
+        # for the deployment to work correctly.
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.6
+      - name: Install dependencies
+        run: |
+          python setup.py install
+      - name: Build Docs
+        run: |
+          cd docs
+          make html
+      - name: Deploy Docs 🚀
+        uses: JamesIves/github-pages-deploy-action@releases/v3
+        with:
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          BRANCH: gh-pages # The branch the action should deploy to.
+          FOLDER: build # The folder the action should deploy.
diff --git a/secedgar/client/network_client.py b/secedgar/client/network_client.py
index 4f728f3..7211a60 100644
--- a/secedgar/client/network_client.py
+++ b/secedgar/client/network_client.py
@@ -85,22 +85,21 @@ class NetworkClient(AbstractClient):
             to request.
 
         Returns:
-            response (requests.response): A requests.response object.
+            response (requests.Response): A `requests.Response` object.
 
         Raises:
             EDGARQueryError: If problems arise when making query.
         """
         prepared_url = self._prepare_query(path)
-        response = None
-        for _ in range(self.retry_count + 1):
-            response = requests.get(url=prepared_url, params=params, **kwargs)
-            if response.status_code == 200:
-                try:
-                    self._validate_response(response)
-                except EDGARQueryError:
-                    continue
-            time.sleep(self.pause)
-        self._validate_response(response)
+        for i in range(self.retry_count + 1):
+            response = requests.get(prepared_url, params=params, **kwargs)
+            try:
+                self._validate_response(response)
+            except EDGARQueryError:
+                time.sleep(self.pause)
+                # Raise query error if on last retry
+                if i == self.retry_count:
+                    raise EDGARQueryError()
         self.response = response
         return self.response
 
@@ -146,3 +145,6 @@ class NetworkClient(AbstractClient):
                                   "your request.")
         elif any(error_message in response.text for error_message in error_messages):
             raise EDGARQueryError()
+        # Need to check for error messages before checking for 200 status code
+        elif status_code != 200:
+            raise EDGARQueryError()
diff --git a/secedgar/filings/cik_validator.py b/secedgar/filings/cik_validator.py
index e81e24a..eb89201 100644
--- a/secedgar/filings/cik_validator.py
+++ b/secedgar/filings/cik_validator.py
@@ -67,25 +67,48 @@ class _CIKValidator(object):
                 pass  # If multiple companies, found, just print out warnings
         return ciks
 
-    def _get_cik(self, lookup):
-        """
-        Get cik for lookup value.
+    def _get_lookup_soup(self, lookup):
+        """Gets `BeautifulSoup` object for lookup.
+
+        *Note: Only to be used internally by `_get_cik` to get CIK from lookup.*
+
+        Args:
+            lookup (str): CIK, company name, or ticker symbol to lookup.
+
+        Returns:
+            soup (bs4.BeautifulSoup): `BeautifulSoup` object to be used to get
+                company CIK.
         """
-        self._validate_lookup(lookup)
         try:  # try to lookup by CIK
             self._params['CIK'] = lookup
-            soup = self._client.get_soup(self.path, self.params)
+            return self._client.get_soup(self.path, self.params)
         except EDGARQueryError:  # fallback to lookup by company name
             del self._params['CIK']  # delete this parameter so no conflicts arise
             self._params['company'] = lookup
-            soup = self._client.get_soup(self.path, self.params)
+            return self._client.get_soup(self.path, self.params)
+
+    def _get_cik(self, lookup):
+        """Gets CIK from `BeautifulSoup` object.
+
+        .. warning: This method will warn when lookup returns multiple possibilities for a
+            CIK are found.
+
+        Args:
+            lookup (str): CIK, company name, or ticker symbol which was looked up.
+
+        Returns:
+            CIK (str): CIK for lookup.
+        """
+        self._validate_lookup(lookup)
+        soup = self._get_lookup_soup(lookup)
         try:  # try to get single CIK for lookup
             span = soup.find('span', {'class': 'companyName'})
             return span.find('a').getText().split()[0]  # returns single CIK
         except AttributeError:  # warn and skip if multiple possibilities for CIK found
-            warnings.warn("Lookup '{0}' will be skipped. "
-                          "Found multiple companies matching '{0}':".format(lookup))
-            warnings.warn('\n'.join(self._get_cik_possibilities(soup)))
+            warning_message = """Lookup '{0}' will be skipped.
+                          Found multiple companies matching '{0}':
+                          {1}""".format(lookup, '\n'.join(self._get_cik_possibilities(soup)))
+            warnings.warn(warning_message)
         finally:
             # Delete parameters after lookup
             if self._params.get('company') is not None:
diff --git a/secedgar/filings/filing.py b/secedgar/filings/filing.py
index b4676e9..a6ddd73 100644
--- a/secedgar/filings/filing.py
+++ b/secedgar/filings/filing.py
@@ -23,11 +23,13 @@ class Filing(AbstractFiling):
         end_date (Union[str, datetime.datetime], optional): Date after which not to fetch reports.
             Stands for "date before." Defaults to today.
 
+    Keyword Args:
+        count (int): Number of filings to get per request. Passed to NetworkClient
+            created on initialization if none given.
+
     .. versionadded:: 0.1.5
     """
 
-    # TODO: Maybe allow NetworkClient to take in kwargs
-    #  (set to None and if None, create NetworkClient with kwargs)
     def __init__(self,
                  cik_lookup,
                  filing_type,
@@ -122,6 +124,9 @@ class Filing(AbstractFiling):
         Args:
             kwargs: Anything to be passed to requests when making get request.
 
+        Keyword Args:
+            See keyword arguments accepted for ``secedgar.client._base.AbstractClient.get_soup``
+
         Returns:
             urls (list): List of urls for txt files to download.
         """
@@ -140,6 +145,9 @@ class Filing(AbstractFiling):
             cik (str): CIK for company.
             kwargs: Anything to be passed to requests when making get request.
 
+        Keyword Args:
+            See keyword arguments accepted for ``secedgar.client._base.AbstractClient.get_soup``
+
         Returns:
             txt_urls (list of str): Up to the desired number of URLs for that specific company
             if available.
diff --git a/secedgar/utils/exceptions.py b/secedgar/utils/exceptions.py
index 5663f5d..703bb1e 100644
--- a/secedgar/utils/exceptions.py
+++ b/secedgar/utils/exceptions.py
@@ -3,7 +3,7 @@ class EDGARQueryError(Exception):
     This error is thrown when a query receives a response that is not a 200 response.
     """
 
-    def __str__(self):
+    def __str__(self):  # pragma: no cover
         return "An error occured while making the query."
 
 
@@ -12,13 +12,13 @@ class EDGARFieldError(Exception):
     This error is thrown when an invalid field is given to an endpoint.
     """
 
-    def __init__(self, endpoint, field):
+    def __init__(self, endpoint, field):  # pragma: no cover
         self.endpoint = endpoint
         self.field = field
 
-    def __str__(self):
+    def __str__(self):  # pragma: no cover
         return "Field {field} not found in endpoint {endpoint}".format(
-                field=self.field, endpoint=self.endpoint
+            field=self.field, endpoint=self.endpoint
         )
 
 
@@ -30,13 +30,13 @@ class CIKError(Exception):
     def __init__(self, cik):
         self.cik = cik
 
-    def __str__(self):
+    def __str__(self):  # pragma: no cover
         return "CIK {cik} not valid.".format(cik=self.cik)
 
 
 class FilingTypeError(Exception):
     """This error is thrown when an invalid filing type is given. """
 
-    def __str__(self):
+    def __str__(self):  # pragma: no cover
         return "The filing type given is not valid. " \
                "Filing type must be in valid filing type from FilingType class"


Test patch:
diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pypi.yml
similarity index 100%
rename from .github/workflows/pythonpublish.yml
rename to .github/workflows/pypi.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..cba2d25
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,36 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Tests
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install python-dev libxml2-dev libxslt-dev libz-dev
+          pip install -qq flake8
+          pip install pytest
+          python setup.py install
+      - name: Lint with flake8
+        run: |
+          set -e
+          flake8 --version
+          flake8 secedgar --count --show-source --statistics
+      - name: Test with pytest
+        run: |
+          pytest secedgar/tests
diff --git a/secedgar/tests/client/test_network_client.py b/secedgar/tests/client/test_network_client.py
index 73c5a70..ae4c5a0 100644
--- a/secedgar/tests/client/test_network_client.py
+++ b/secedgar/tests/client/test_network_client.py
@@ -27,6 +27,7 @@ class MockBadStatusCodeResponse:
         if (status_code == 200):
             raise ValueError("status_code should not equal 200.")
         self.status_code = status_code
+        self.text = ""
 
     def __call__(self, *args, **kwargs):
         return self
@@ -68,7 +69,7 @@ class MockSingleFilingPageGoodResponse:
             self.text = f.read()
 
 
-class TestClient:
+class TestNetworkClient:
     def test_client_bad_response_raises_error(self, monkeypatch, client):
         monkeypatch.setattr(requests, 'get', MockNoCIKFoundBadResponse)
         with pytest.raises(EDGARQueryError):
@@ -87,17 +88,18 @@ class TestClient:
         assert client.get_response('path', {})
 
     @pytest.mark.parametrize(
-        "status_code,expectation",
+        "status_code",
         [
-            (400, pytest.raises(EDGARQueryError)),
-            (401, pytest.raises(EDGARQueryError)),
-            (500, pytest.raises(EDGARQueryError)),
-            (501, pytest.raises(EDGARQueryError))
+            203,
+            400,
+            401,
+            500,
+            501
         ]
     )
-    def test_client_bad_response_codes(self, status_code, expectation, monkeypatch, client):
+    def test_client_bad_response_codes(self, status_code, monkeypatch, client):
         monkeypatch.setattr(requests, 'get', MockBadStatusCodeResponse(status_code))
-        with expectation:
+        with pytest.raises(EDGARQueryError):
             client.get_response('path', {})
 
     @pytest.mark.parametrize(
diff --git a/secedgar/tests/filings/test_cik_lookup.py b/secedgar/tests/filings/test_cik_lookup.py
index bcfc6b5..51bc183 100644
--- a/secedgar/tests/filings/test_cik_lookup.py
+++ b/secedgar/tests/filings/test_cik_lookup.py
@@ -34,10 +34,15 @@ class TestCIKLookup(object):
         monkeypatch.setattr(NetworkClient, "get_response", MockSingleCIKLookupResponse)
         assert aapl.ciks == ['0000320193']
 
+    def test_cik_lookup_lookups_property(self):
+        multiple_company_lookup = CIKLookup(['aapl', 'msft', 'fb'])
+        assert multiple_company_lookup.lookups == ['aapl', 'msft', 'fb']
+
     def test_multiple_results_company_name_search(self, monkeypatch):
         multiple_results_cik = CIKLookup('paper')
         monkeypatch.setattr(NetworkClient, "get_response", MockSingleCIKMultipleResultsResponse)
-        assert len(multiple_results_cik.ciks) == 0
+        with pytest.warns(UserWarning):
+            assert len(multiple_results_cik.ciks) == 0
 
     def test_multiple_results_raises_warnings(self, monkeypatch):
         multiple_results_cik = CIKLookup('paper')
diff --git a/secedgar/tests/filings/test_cik_validator.py b/secedgar/tests/filings/test_cik_validator.py
index 258cad0..12f9212 100644
--- a/secedgar/tests/filings/test_cik_validator.py
+++ b/secedgar/tests/filings/test_cik_validator.py
@@ -65,5 +65,4 @@ class TestCIKValidator:
     def test_params_reset_after_get_cik(self, ticker_lookups, client):
         validator = _CIKValidator(lookups=ticker_lookups, client=client)
         validator._get_cik(ticker_lookups[0])
-        assert validator.params.get("CIK") is None
-        assert validator.params.get("company") is None
+        assert validator.params.get("CIK") is None and validator.params.get("company") is None
diff --git a/secedgar/tests/filings/test_filings.py b/secedgar/tests/filings/test_filings.py
index e3e1d01..e50d698 100644
--- a/secedgar/tests/filings/test_filings.py
+++ b/secedgar/tests/filings/test_filings.py
@@ -52,9 +52,9 @@ class TestFiling(object):
         if len(urls) != aapl.client.count:
             raise AssertionError("""Count should return exact number of filings.
                                  Got {0}, but expected {1} URLs.""".format(
-                    urls, aapl.client.count))
+                urls, aapl.client.count))
 
-    def test_date_is_sanitized(self, monkeypatch):
+    def test_date_is_sanitized(self):
         start_date = datetime.datetime(2012, 3, 1)
         end_date = datetime.datetime(2015, 1, 1)
         aapl = Filing(cik_lookup='aapl',
@@ -77,7 +77,20 @@ class TestFiling(object):
         assert aapl.start_date == datetime.datetime(2010, 1, 1)
         assert aapl.params['datea'] == '20100101'
 
-    # TODO: Monkeypatch with example response
+    @pytest.mark.parametrize(
+        "date,expected",
+        [
+            ("20120101", "20120101"),
+            (20120101, 20120101),
+            (datetime.datetime(2012, 1, 1), "20120101")
+        ]
+    )
+    def test_end_date_setter(self, date, expected):
+        f = Filing('aapl', FilingType.FILING_10Q, start_date=datetime.datetime(
+            2010, 1, 1), end_date=datetime.datetime(2015, 1, 1))
+        f.end_date = date
+        assert f.end_date == date and f.params.get("dateb") == expected
+
     @pytest.mark.slow
     def test_txt_urls(self, monkeypatch):
         aapl = Filing(cik_lookup='aapl', filing_type=FilingType.FILING_10Q, count=10)
@@ -90,22 +103,79 @@ class TestFiling(object):
         with pytest.raises(AttributeError):
             Filing(cik_lookup='0000320193', filing_type=FilingType.INVALID)
 
-    def test_invalid_filing_type_types(self):
-        for t in ('10j', '10-k', 'ssd', 'invalid', 1):
-            with pytest.raises(FilingTypeError):
-                Filing(cik_lookup='0000320193', filing_type=t)
-
-    def test_validate_cik_type_inside_filing(self):
+    @pytest.mark.parametrize(
+        "new_filing_type",
+        (
+            FilingType.FILING_10K,
+            FilingType.FILING_8K,
+            FilingType.FILING_13F,
+            FilingType.FILING_SD
+        )
+    )
+    def test_filing_type_setter(self, new_filing_type):
+        f = Filing(cik_lookup='aapl', filing_type=FilingType.FILING_10Q)
+        f.filing_type = new_filing_type
+        assert f.filing_type == new_filing_type
+
+    @pytest.mark.parametrize(
+        "bad_filing_type",
+        (
+            '10-k',
+            '10k',
+            '10-q',
+            '10q',
+            123
+        )
+    )
+    def test_bad_filing_type_setter(self, bad_filing_type):
+        f = Filing(cik_lookup='aapl', filing_type=FilingType.FILING_10Q)
+        with pytest.raises(FilingTypeError):
+            f.filing_type = bad_filing_type
+
+    @pytest.mark.parametrize(
+        "bad_filing_type",
+        (
+            "10-j",
+            "10-k",
+            "ssd",
+            "invalid",
+            1
+        )
+    )
+    def test_invalid_filing_type_types(self, bad_filing_type):
+        with pytest.raises(FilingTypeError):
+            Filing(cik_lookup='0000320193', filing_type=bad_filing_type)
+
+    @pytest.mark.parametrize(
+        "bad_cik_lookup",
+        (
+            1234567891011,
+            12345,
+            123.0
+        )
+    )
+    def test_validate_cik_type_inside_filing(self, bad_cik_lookup):
         with pytest.raises(TypeError):
-            Filing(cik_lookup=1234567891011, filing_type=FilingType.FILING_10K)
-        with pytest.raises(TypeError):
-            Filing(cik_lookup=123.0, filing_type=FilingType.FILING_10K)
+            Filing(cik_lookup=bad_cik_lookup, filing_type=FilingType.FILING_10K)
 
     def test_validate_cik_inside_filing(self, monkeypatch):
         monkeypatch.setattr(NetworkClient, "get_response", MockSingleCIKNotFound)
         with pytest.raises(EDGARQueryError):
             _ = Filing(cik_lookup='0notvalid0', filing_type=FilingType.FILING_10K).cik_lookup.ciks
 
+    @pytest.mark.parametrize(
+        "no_urls",
+        (
+            {},
+            {'aapl': [], 'fb': [], 'msft': []}
+        )
+    )
+    def test_save_no_filings_raises_error(self, tmp_data_directory, monkeypatch, no_urls):
+        monkeypatch.setattr(Filing, "get_urls", lambda x: no_urls)
+        f = Filing(cik_lookup='aapl', filing_type=FilingType.FILING_10K)
+        with pytest.raises(ValueError):
+            f.save(tmp_data_directory)
+
     def test_filing_save_multiple_ciks(self, tmp_data_directory, monkeypatch):
         monkeypatch.setattr(_CIKValidator, "get_ciks", MockCIKValidatorMultipleCIKs.get_ciks)
         monkeypatch.setattr(NetworkClient, "get_response", MockSingleCIKFiling)
diff --git a/secedgar/tests/test_utils.py b/secedgar/tests/test_utils.py
new file mode 100644
index 0000000..2fecb26
--- /dev/null
+++ b/secedgar/tests/test_utils.py
@@ -0,0 +1,46 @@
+from datetime import datetime
+import pytest
+
+from secedgar.utils import sanitize_date
+
+
+class TestUtils:
+    @pytest.mark.parametrize(
+        "bad_date",
+        [
+            "2012101",
+            "201210",
+            "1010",
+            "2012011",
+            2012011,
+            2012101,
+            2012,
+            1010,
+            201210
+        ]
+    )
+    def test_bad_date_formats(self, bad_date):
+        with pytest.raises(TypeError):
+            sanitize_date(bad_date)
+
+    @pytest.mark.parametrize(
+        "good_date",
+        [
+            "20120101",
+            20120101
+        ]
+    )
+    def test_good_formats_no_change(self, good_date):
+        """Tests formats that should not change from what is given. """
+        assert sanitize_date(good_date) == good_date
+
+    @pytest.mark.parametrize(
+        "dt_date,expected",
+        [
+            (datetime(2018, 1, 1), "20180101"),
+            (datetime(2020, 3, 4), "20200304"),
+            (datetime(2020, 7, 18), "20200718")
+        ]
+    )
+    def test_good_formats_datetime(self, dt_date, expected):
+        assert sanitize_date(dt_date) == expected
diff --git a/test.sh b/test.sh
new file mode 100755
index 0000000..3365f20
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+# run tests, omit tests
+coverage run --source secedgar --omit=*/tests* -m pytest
+# output coverage results to xml file
+coverage xml -o cov.xml
+# Show missing lines
+coverage report -m
\ No newline at end of file