Repository: sec-edgar/sec-edgar

Issue:
Add compatibility with EDGAR APIs
**Is your feature request related to a problem? Please describe.**
The SEC is currently offering a RESTful API in beta! See more [here](https://www.sec.gov/edgar/sec-api-documentation). Not sure exactly what the best way to integrate this into the package would be, but want to put it out there so that it can be discussed.

**Describe the solution you'd like**
Not sure yet. Add an api package that provides utilities to easily access the API? Should be able to reuse `NetworkClient` code.


Comments:

Patch:
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 776f001..a3b7971 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -16,6 +16,7 @@ Welcome to secedgar's documentation!
    ciklookup
    filings
    filingtypes
+   rest_api
    cikmap
    client
    parser
diff --git a/docs/source/rest_api.rst b/docs/source/rest_api.rst
new file mode 100644
index 0000000..fdb2961
--- /dev/null
+++ b/docs/source/rest_api.rst
@@ -0,0 +1,24 @@
+.. _rest_api:
+
+REST API
+========
+
+The SEC recently released an API which can be used to more easily access data on companies.
+For a complete overview of the API, you can
+`read more here <https://www.sec.gov/edgar/sec-api-documentation>`_.
+
+
+Wrapper
+-------
+
+In order to make this information even easier to extract, ``secedgar`` contains functions that
+wrap around the EDGAR API to make the data even more accessible. Below are the functions available
+via ``secedgar``.
+
+.. autofunction:: secedgar.core.rest.get_submissions
+
+.. autofunction:: secedgar.core.rest.get_company_concepts
+
+.. autofunction:: secedgar.core.rest.get_company_facts
+
+.. autofunction:: secedgar.core.rest.get_xbrl_frames
\ No newline at end of file
diff --git a/docs/source/whatsnew.rst b/docs/source/whatsnew.rst
index bbe2b34..1673705 100644
--- a/docs/source/whatsnew.rst
+++ b/docs/source/whatsnew.rst
@@ -6,6 +6,10 @@ What's New
 
 New features, bug fixes, and improvements for each release.
 
+.. include:: whatsnew/v0.5.0.rst
+
+.. include:: whatsnew/v0.4.1.rst
+
 .. include:: whatsnew/v0.4.0.rst
 
 .. include:: whatsnew/v0.3.4.rst
diff --git a/docs/source/whatsnew/v0.5.0.rst b/docs/source/whatsnew/v0.5.0.rst
new file mode 100644
index 0000000..b4f1550
--- /dev/null
+++ b/docs/source/whatsnew/v0.5.0.rst
@@ -0,0 +1,14 @@
+v0.5.0
+------
+
+Highlights
+~~~~~~~~~~
+
+- Add access to SEC's REST API via ``secedgar.core.rest`` module
+- New functions include ``get_submissions``, ``get_company_concepts``, ``get_company_facts``, and ``get_xbrl_frames``.
+- Set ``client`` keyword arg to ``None`` by default for ``CIKLookup``. This will create client object automatically instead of requiring user to provide client.
+
+Contributors
+~~~~~~~~~~~~
+
+- jackmoody11
\ No newline at end of file
diff --git a/secedgar/__init__.py b/secedgar/__init__.py
index 89c9884..ca7abca 100644
--- a/secedgar/__init__.py
+++ b/secedgar/__init__.py
@@ -1,4 +1,4 @@
-__version__ = '0.4.1'
+__version__ = '0.5.0'
 
 from secedgar.core.combo import ComboFilings  # noqa: F401
 from secedgar.core.company import CompanyFilings  # noqa: F401
diff --git a/secedgar/cik_lookup.py b/secedgar/cik_lookup.py
index 4b51964..167d522 100644
--- a/secedgar/cik_lookup.py
+++ b/secedgar/cik_lookup.py
@@ -43,13 +43,19 @@ class CIKLookup:
 
     Args:
         lookup (Union[str, list]): Ticker, company name, or list of tickers and/or company names.
-        client (secedgar.client.NetworkClient): A network client object to use. See
-            :class:`secedgar.client.NetworkClient` for more details.
+        client (Union[None, secedgar.client.NetworkClient]): A network client object to use.
+            If ``None`` is given, then will pass kwargs to :class:`secedgar.client.NetworkClient`.
+            See :class:`secedgar.client.NetworkClient` for more details. Defaults to ``None``.
+
+    .. warning::
+
+        If ``None`` is passed to client, you must specify your ``user_agent`` in ``kwargs``.
+        For example, ``CIKLookup(lookup=["aapl"], user_agent="Name (email@example.com")``.
 
     .. versionadded:: 0.1.5
     """
 
-    def __init__(self, lookups, client, **kwargs):
+    def __init__(self, lookups, client=None, **kwargs):
         if lookups and isinstance(lookups, str):
             self._lookups = [lookups]  # make single string into list
         else:
diff --git a/secedgar/core/rest.py b/secedgar/core/rest.py
new file mode 100644
index 0000000..766e406
--- /dev/null
+++ b/secedgar/core/rest.py
@@ -0,0 +1,205 @@
+from typing import List, Union
+
+import requests
+
+from secedgar.cik_lookup import CIKLookup
+
+API_BASE = "https://data.sec.gov/api/"
+XBRL_BASE = "{0}xbrl/".format(API_BASE)
+
+
+def _get_lookup_dict(lookups: List[str], user_agent: str):
+    """Utility function to get CIKs for lookups.
+
+    Args:
+        lookups (list of str): List of tickers or company names to get CIKs for.
+        user_agent (str): User agent to give to SEC.
+
+    Returns:
+        dict: Dictionary with lookup and CIK key-value pairs.
+    """
+    cik_lookup = CIKLookup(lookups=lookups,
+                           user_agent=user_agent,
+                           client=None)
+    return cik_lookup.lookup_dict
+
+
+def _combine_dicts(*dicts):
+    """Utility function to combine dictionary values when values are lists.
+
+    Returns:
+        dict: Dictionaries combined into one.
+
+    Examples:
+        >>> a = {"A": [1, 2, 3], "B": [4, 5, 6]}
+        >>> b = {"A": [7, 8], "B": [0, 1, 2]}
+        >>> _combine_dicts(a, b)
+        {'A': [1, 2, 3, 7, 8], 'B': [4, 5, 6, 0, 1, 2]}
+        >>> _combine_dicts(a)
+        {'A': [1, 2, 3], 'B': [4, 5, 6]}
+    """
+    final = {}
+    for d in dicts:
+        for k, v in d.items():
+            if k in final:
+                final[k] += v
+            else:
+                final[k] = v
+    return final
+
+
+def get_submissions(lookups: Union[List[str], str],
+                    user_agent: str,
+                    recent: bool = True) -> dict:
+    """Get information about submissions for entities.
+
+    Args:
+        lookups (list of str or str): Tickers or CIKs to get submission data for.
+        user_agent (str): User agent to provide the SEC.
+        recent (bool, optional): Whether or not to only get the most recent.
+            Setting ``recent`` to True will give at least one year of filings or 1000 filings
+            (whichever is more). Setting ``recent`` to False will return all filings.
+            Defaults to True.
+
+    Returns:
+        dict: Dictionary with keys being the lookups and values being the responses from the API.
+    """
+    lookup_dict = _get_lookup_dict(lookups=lookups, user_agent=user_agent)
+    submissions_dict = dict()
+    url_base = "https://data.sec.gov/submissions/"
+    for lookup, cik in lookup_dict.items():
+        resp = requests.get("{0}CIK{1}.json".format(url_base, cik.zfill(10)),
+                            headers={"user-agent": user_agent})
+        resp_json = resp.json()
+        if not recent:
+            try:
+                older_submission_files = resp_json["filings"]["files"]
+            except KeyError:
+                pass
+
+            # Get data for older submission files and add to recent
+            older_submissions = [requests.get("{0}{1}".format(url_base, f["name"]),
+                                              headers={"user-agent": user_agent}).json()
+                                 for f in older_submission_files]
+
+            resp_json["filings"]["recent"] = _combine_dicts(resp_json["filings"]["recent"],
+                                                            *older_submissions)
+        submissions_dict[lookup] = resp_json
+    return submissions_dict
+
+
+def get_company_concepts(lookups: Union[List[str], str],
+                         user_agent: str,
+                         concept_name: str) -> dict:
+    """Get company concepts using SEC's REST API.
+
+    Args:
+        lookups (list of str or str): Tickers or CIKs to get concepts for.
+        user_agent (str): User agent to send to SEC.
+        concept_name (str): Name of the concept to get data for.
+
+    Returns:
+        dict: Dictionary with concept data for given lookups.
+
+    Example:
+        .. code::
+
+            concept = "AccountsPayableCurrent"
+            get_company_concepts(lookups=["AAPL"],
+                                 user_agent="Name (email@example.com)",
+                                 concept_name=concept)
+    """
+    lookup_dict = _get_lookup_dict(lookups=lookups, user_agent=user_agent)
+    company_concepts = dict()
+    for lookup, cik in lookup_dict.items():
+        url = "{0}companyconcept/CIK{1}/us-gaap/{2}.json".format(
+            XBRL_BASE,
+            cik.zfill(10),
+            concept_name
+        )
+        resp = requests.get(url,
+                            headers={"user-agent": user_agent})
+        company_concepts[lookup] = resp.json()
+    return company_concepts
+
+
+def get_company_facts(lookups: Union[List[str], str], user_agent: str) -> dict:
+    """Get company facts for lookups.
+
+    Args:
+        lookups (list of str or str): Tickers or CIKs to get company facts for.
+        user_agent (str): User agent to send to SEC.
+
+    Returns:
+        dict: Dictionary with lookups as keys and company fact dictionaries as values.
+
+    Examples:
+        >>> facts = get_company_facts(["aapl"], "Name (email@example.com)")
+        >>> single_fact = facts["aapl"]["facts"]["us-gaap"]["Assets"]["units"]["USD"][0]
+        >>> single_fact["val"]
+        39572000000
+        >>> single_fact["fy"]
+        2009
+        >>> single_fact["fp"]
+        Q3
+        >>> single_fact["form"]
+        '10-Q'
+        >>> single_fact["filed"]
+        '2009-07-22'
+    """
+    lookup_dict = _get_lookup_dict(lookups=lookups, user_agent=user_agent)
+    company_facts = dict()
+    for lookup, cik in lookup_dict.items():
+        url = "{0}companyfacts/CIK{1}.json".format(XBRL_BASE, cik.zfill(10))
+        resp = requests.get(url, headers={"user-agent": user_agent})
+        company_facts[lookup] = resp.json()
+    return company_facts
+
+
+def get_xbrl_frames(user_agent: str,
+                    concept_name: str,
+                    year: int,
+                    quarter: Union[None, int] = None,
+                    currency: str = "USD",
+                    instantaneous: bool = False) -> dict:
+    """Get data for concept name in year (and quarter, if given).
+
+    Args:
+        user_agent (str): User agent to send to the SEC.
+        concept_name (str): Concept name to get. For example, "Assets".
+        year (int): Year to get concept data for.
+        quarter (Union[int, NoneType], optional): Quarter to get data for. If given None,
+            will look for data for the entire year. Defaults to None.
+        instantaneous (bool, optional): Whether to look for instantaneous data.
+            See `SEC website for more <https://www.sec.gov/edgar/sec-api-documentation>`_.
+            Defaults to False.
+
+    Returns:
+        dict: Dictionary with information about concept_name.
+            Dig into the data key for all period data.
+
+    Examples:
+        .. code::
+
+            frames = get_xbrl_frames(user_agent="Name (email@example.com)",
+                                     concept_name="Assets",
+                                     year=2020,
+                                     quarter=3,
+                                     instantaneous=True)
+            print(frames["data"][-1]["entityName"])
+            # Prints "MOXIAN (BVI) INC"
+            print(frames["data"][-1]["val"])
+            # Prints 2295657
+            print(frames["data"][-1]["accn"])
+            # Prints "0001493152-22-013323"
+    """
+    # Create URL
+    period = "CY{0}".format(year) if quarter is None else "CY{0}Q{1}".format(year, quarter)
+    if instantaneous:
+        period += "I"
+    url = "{0}frames/us-gaap/{1}/{2}/{3}.json".format(XBRL_BASE, concept_name, currency, period)
+    print(url)
+    # Request and add to dictionary
+    resp = requests.get(url, headers={"user-agent": user_agent})
+    xbrl_frames = resp.json()
+    return xbrl_frames


Test patch:
diff --git a/secedgar/tests/core/test_company.py b/secedgar/tests/core/test_company.py
index ec7e80c..26c1375 100644
--- a/secedgar/tests/core/test_company.py
+++ b/secedgar/tests/core/test_company.py
@@ -3,6 +3,7 @@ import datetime
 import os
 
 import pytest
+
 from secedgar.cik_lookup import CIKLookup
 from secedgar.client import NetworkClient
 from secedgar.core import CompanyFilings, FilingType
@@ -291,6 +292,45 @@ class TestCompanyFilings:
         f.save(tmp_data_directory)
         assert len(os.listdir(tmp_data_directory)) > 0
 
+    @pytest.mark.parametrize(
+        "match_format",
+        [
+            "EXACT",
+            "AMEND",
+            "ALL"
+        ]
+    )
+    def test_match_format_good(self, mock_user_agent, match_format):
+        f = CompanyFilings(["aapl", "amzn", "msft"],
+                           FilingType.FILING_10Q,
+                           user_agent=mock_user_agent,
+                           count=3,
+                           match_format=match_format)
+
+        assert f.match_format == match_format
+
+    @pytest.mark.parametrize(
+        "match_format",
+        [
+            "exact",
+            "amend",
+            "all",
+            "none",
+            None,
+            True,
+            False,
+            1,
+            0
+        ]
+    )
+    def test_match_format_bad(self, mock_user_agent, match_format):
+        with pytest.raises(ValueError):
+            CompanyFilings(["aapl", "amzn", "msft"],
+                           FilingType.FILING_10Q,
+                           user_agent=mock_user_agent,
+                           count=3,
+                           match_format=match_format)
+
     @pytest.mark.smoke
     def test_filing_save_multiple_ciks_smoke(self, tmp_data_directory,
                                              real_test_client):
diff --git a/secedgar/tests/core/test_rest.py b/secedgar/tests/core/test_rest.py
new file mode 100644
index 0000000..743763f
--- /dev/null
+++ b/secedgar/tests/core/test_rest.py
@@ -0,0 +1,104 @@
+import pytest
+
+from secedgar.core.rest import (_combine_dicts, get_company_concepts,
+                                get_company_facts, get_submissions,
+                                get_xbrl_frames)
+
+
+class TestRest:
+    @pytest.mark.parametrize("dicts,expected",
+                             [
+                                 ([{"A": [1, 2, 3], "B": [4, 5, 6]},
+                                  {"A": [7, 8], "B": [0, 1, 2]}],
+                                  {'A': [1, 2, 3, 7, 8], 'B': [4, 5, 6, 0, 1, 2]}),
+                                 ([{"A": [1, 2, 3]},
+                                  {"B": [4, 5, 6]}],
+                                  {"A": [1, 2, 3], "B": [4, 5, 6]}),
+                                 ([{"A": [1]}],
+                                  {"A": [1]})
+                             ]
+                             )
+    def test__combine_dicts(self, dicts, expected):
+        assert _combine_dicts(*dicts) == expected
+
+    @pytest.mark.parametrize(
+        "recent",
+        [True, False]
+    )
+    @pytest.mark.smoke
+    def test_get_submissions(self, mock_user_agent, recent):
+        submissions = get_submissions(lookups=["aapl"],
+                                      user_agent=mock_user_agent,
+                                      recent=recent)
+        assert submissions
+        # Make sure Apple's CIK shows up properly
+        assert str(submissions["aapl"]["cik"]) == "320193"
+
+        # Make sure there are accession numbers
+        assert submissions["aapl"]["filings"]["recent"]["accessionNumber"]
+
+        # Result should be dictionary
+        assert isinstance(submissions, dict)
+
+    @pytest.mark.smoke
+    def test_get_company_concepts(self, mock_user_agent):
+        concept = "AccountsPayableCurrent"
+        concepts = get_company_concepts(lookups=["AAPL"],
+                                        user_agent=mock_user_agent,
+                                        concept_name=concept)
+        assert concepts
+        # Ensure CIK is correct
+        assert str(concepts["AAPL"]["cik"]) == "320193"
+
+        # Make sure that there are results for accounts payable
+        assert concepts["AAPL"]["units"]["USD"]
+
+        # Result should be dictionary
+        assert isinstance(concepts, dict)
+
+    @pytest.mark.smoke
+    def test_get_company_facts(self, mock_user_agent):
+        facts = get_company_facts(lookups=["aapl"], user_agent=mock_user_agent)
+
+        assert facts
+        # Ensure CIK is correct - sometimes will give number, so cast to string
+        assert str(facts["aapl"]["cik"]) == "320193"
+
+        # Make sure there are facts
+        assert facts["aapl"]["facts"]
+
+        # Make sure that us-gaap and dei both are keys
+        assert "us-gaap" in facts["aapl"]["facts"]
+        assert "dei" in facts["aapl"]["facts"]
+
+        # Make sure we can get Revenues for Apple
+        assert facts["aapl"]["facts"]["us-gaap"]["Revenues"]["units"]["USD"]
+
+        # Result should be dictionary
+        assert isinstance(facts, dict)
+
+    @pytest.mark.smoke
+    @pytest.mark.parametrize(
+        "concept,instantaneous",
+        [
+            ("Revenues", False),
+            ("Revenues", True)
+        ]
+    )
+    def test_get_xbrl_frames(self, mock_user_agent, concept, instantaneous):
+        frames = get_xbrl_frames(user_agent=mock_user_agent,
+                                 concept_name=concept,
+                                 year=2020,
+                                 quarter=1,
+                                 instantaneous=instantaneous)
+        # Check to make sure we got the right frame
+        assert frames["tag"] == concept
+
+        # Make sure there is data
+        assert frames["data"]
+
+        # Data should have accn and cik as keys
+        assert "accn" in frames["data"][0]
+        assert "cik" in frames["data"][0]
+
+        assert isinstance(frames, dict)
diff --git a/secedgar/tests/test_cik_lookup.py b/secedgar/tests/test_cik_lookup.py
index 165c229..5587ac8 100644
--- a/secedgar/tests/test_cik_lookup.py
+++ b/secedgar/tests/test_cik_lookup.py
@@ -235,3 +235,8 @@ class TestCIKLookup(object):
         cik_map = get_cik_map()
         assert "ticker" in cik_map and "title" in cik_map
         assert cik_map["ticker"] and cik_map["title"]
+
+    def test_cik_lookup_client_none(self, ticker_lookups, mock_user_agent):
+        lookup = CIKLookup(lookups=ticker_lookups, user_agent=mock_user_agent, client=None)
+        assert isinstance(lookup.client, NetworkClient)
+        assert lookup.lookups == ticker_lookups