Repository: sec-edgar/sec-edgar

Issue:
Form 4 Parser
**Is your feature request related to a problem? Please describe.**
Insider trading information has become a hot commodity for retail investors. The parser does not currently extract pertinent data on buying and selling of securities by insiders.

**Describe the solution you'd like**
Add a function to the parser to extract non-derivative transaction information pertinent to retail investors intending to track trades of insiders.

**Describe alternatives you've considered**
All other alternatives are closed-source/proprietary and paid.

**Additional context**
No additional context.


Comments:

Patch:
diff --git a/docs/source/parser.rst b/docs/source/parser.rst
index a13e72b..d684367 100644
--- a/docs/source/parser.rst
+++ b/docs/source/parser.rst
@@ -16,4 +16,9 @@ Parsing Meta Data
 -----------------
 
 .. autoclass:: secedgar.parser.MetaParser
+   :members:
+
+Parsing Form 4 Data 
+
+.. autoclass:: secedgar.parser.F4Parser 
    :members:
\ No newline at end of file
diff --git a/secedgar/parser.py b/secedgar/parser.py
index bbb0a4c..cd3cb6b 100644
--- a/secedgar/parser.py
+++ b/secedgar/parser.py
@@ -4,8 +4,26 @@ import os
 import re
 import uu
 
+from secedgar.exceptions import FilingTypeError
 from secedgar.utils import make_path
 
+value_pattern = r"<value>(.*?)</value>"
+non_derivative_trans_pattern = r"<nonDerivativeTransaction>(.*?)</nonDerivativeTransaction>"
+sec_title_pattern = r"<securityTitle>(.*?)</securityTitle>"
+trans_date_pattern = r"<transactionDate>(.*?)</transactionDate>"
+trans_shares_pattern = r"<transactionShares>(.*?)</transactionShares>"
+trans_pps_pattern = r"<transactionPricePerShare>(.*?)</transactionPricePerShare>"
+trans_disp_code_pattern = (
+    r"<transactionAcquiredDisposedCode>"
+    r"(.*?)"
+    r"</transactionAcquiredDisposedCode>"
+    )
+soft_pattern = r"<sharesOwnedFollowingTransaction>(.*?)</sharesOwnedFollowingTransaction>"
+doio_pattern = r"<directOrIndirectOwnership>(.*?)</directOrIndirectOwnership>"
+trans_form_type_pattern = r"<transactionFormType>(.*?)</transactionFormType>"
+trans_code_pattern = r"<transactionCode>(.*?)</transactionCode>"
+equity_swap_involved_pattern = r"<equitySwapInvolved>(.*?)</equitySwapInvolved>"
+
 
 class MetaParser:
     """Utility class to extract metadata and documents from a single text file.
@@ -203,6 +221,7 @@ class MetaParser:
 
         Return:
             dict: Dictionary with fields parsed from document.
+
         """
         metadata_doc = {}
 
@@ -221,3 +240,100 @@ class MetaParser:
         metadata_doc["filename"] = fn_m.group(1)
 
         return metadata_doc
+
+
+class F4Parser:
+    """Utility class to extract actionable data and documents from a single text file.
+
+    .. warning::
+        The ``F4Parser`` class is still experimental. Use with caution.
+
+    .. versionadded:: 0.4.0
+    """
+
+    @staticmethod
+    def process(doc):
+        """Process the actionable data of the document.
+
+        Args:
+            doc (str): Document from which to extract core data.
+
+        Return:
+            data (dict): Tradable buy/sell/gift data from document.
+
+        """
+        metadata = MetaParser.process_document_metadata(doc)
+
+        if metadata["type"] == "4":
+            # Regex find all nested values.
+            def nested_findall(parent_pattern, doc, child_pattern=value_pattern):
+                matches = [
+                    re.search(child_pattern, match).group(1)
+                    for match
+                    in re.findall(parent_pattern, doc, re.S)]
+                return matches
+
+            # Find core data from document.
+            security_title_matches = nested_findall(sec_title_pattern, doc)
+            trans_date_matches = nested_findall(trans_date_pattern, doc)
+            trans_shares_matches = nested_findall(trans_shares_pattern, doc)
+            trans_pps_matches = nested_findall(trans_pps_pattern, doc)
+            trans_disp_code_matches = nested_findall(trans_disp_code_pattern, doc)
+            soft_matches = nested_findall(soft_pattern, doc)
+            doio_matches = nested_findall(doio_pattern, doc)
+            trans_form_matches = re.findall(trans_form_type_pattern, doc)
+            trans_code_matches = re.findall(trans_code_pattern, doc)
+            equity_swap_matches = re.findall(equity_swap_involved_pattern, doc)
+
+            # Map core data to dict
+            data = {
+                "nonDerivativeTable": {
+                    "nonDerivativeTransaction": [
+                        {
+                            "securityTitle": securityTitle,
+                            "transactionDate": transactionDate,
+                            "transactionCoding": {
+                                "transactionFormType": transactionFormType,
+                                "transactionCode": transactionCode,
+                                "equitySwapInvolved": equitySwapInvolved
+                            },
+                            "transactionAmounts": {
+                                "transactionShares": transactionShares,
+                                "transactionPricePerShare": transactionPricePerShare,
+                                "transactionAcquiredDisposedCode": transactionAcquiredDisposedCode
+                            },
+                            "postTransactionAmounts": {
+                                "sharesOwnedFollowingTransaction": sharesOwnedFollowingTransaction
+                            },
+                            "ownershipNature": {
+                                "directOrIndirectOwnership": directOrIndirectOwnership
+                            }
+                        }
+                        for securityTitle,
+                        transactionDate,
+                        transactionFormType,
+                        transactionCode,
+                        equitySwapInvolved,
+                        transactionShares,
+                        transactionPricePerShare,
+                        transactionAcquiredDisposedCode,
+                        sharesOwnedFollowingTransaction,
+                        directOrIndirectOwnership
+                        in zip(
+                            security_title_matches,
+                            trans_date_matches,
+                            trans_form_matches,
+                            trans_code_matches,
+                            equity_swap_matches,
+                            trans_shares_matches,
+                            trans_pps_matches,
+                            trans_disp_code_matches,
+                            soft_matches,
+                            doio_matches
+                        )
+                    ]
+                }
+            }
+            return data
+        else:
+            raise FilingTypeError


Test patch:
diff --git a/secedgar/tests/test_parser.py b/secedgar/tests/test_parser.py
index 6528f8a..a6407d3 100644
--- a/secedgar/tests/test_parser.py
+++ b/secedgar/tests/test_parser.py
@@ -1,5 +1,5 @@
 import pytest
-from secedgar.parser import MetaParser
+from secedgar.parser import F4Parser, MetaParser
 
 
 class TestParser:
@@ -26,3 +26,136 @@ class TestParser:
     def test_bad_filetypes_raises_error(self, bad_filetype):
         with pytest.raises(ValueError):
             self.parser.process(infile="test.{0}".format(bad_filetype))
+
+
+class TestF4Parser:
+
+    parser = F4Parser()
+
+    def test_process_document_metadata_form_4(self):
+        doc = """<TYPE>4
+        <SEQUENCE>123
+        <FILENAME>test-filename.txt
+        <nonDerivativeTable>
+        <nonDerivativeTransaction>
+            <securityTitle>
+                <value>Common Stock</value>
+            </securityTitle>
+            <transactionDate>
+                <value>2021-05-14</value>
+            </transactionDate>
+            <transactionCoding>
+                <transactionFormType>5</transactionFormType>
+                <transactionCode>G</transactionCode>
+                <equitySwapInvolved>0</equitySwapInvolved>
+            </transactionCoding>
+            <transactionTimeliness>
+                <value>E</value>
+            </transactionTimeliness>
+            <transactionAmounts>
+                <transactionShares>
+                    <value>4010</value>
+                </transactionShares>
+                <transactionPricePerShare>
+                    <value>0</value>
+                </transactionPricePerShare>
+                <transactionAcquiredDisposedCode>
+                    <value>D</value>
+                </transactionAcquiredDisposedCode>
+            </transactionAmounts>
+            <postTransactionAmounts>
+                <sharesOwnedFollowingTransaction>
+                    <value>324164</value>
+                </sharesOwnedFollowingTransaction>
+            </postTransactionAmounts>
+            <ownershipNature>
+                <directOrIndirectOwnership>
+                    <value>D</value>
+                </directOrIndirectOwnership>
+            </ownershipNature>
+        </nonDerivativeTransaction>
+        <nonDerivativeTransaction>
+            <securityTitle>
+                <value>Common Stock</value>
+                <footnoteId id="F1"/>
+            </securityTitle>
+            <transactionDate>
+                <value>2021-08-02</value>
+            </transactionDate>
+            <transactionCoding>
+                <transactionFormType>4</transactionFormType>
+                <transactionCode>S</transactionCode>
+                <equitySwapInvolved>0</equitySwapInvolved>
+            </transactionCoding>
+            <transactionAmounts>
+                <transactionShares>
+                    <value>15600</value>
+                </transactionShares>
+                <transactionPricePerShare>
+                    <value>145.83</value>
+                    <footnoteId id="F2"/>
+                </transactionPricePerShare>
+                <transactionAcquiredDisposedCode>
+                    <value>D</value>
+                </transactionAcquiredDisposedCode>
+            </transactionAmounts>
+            <postTransactionAmounts>
+                <sharesOwnedFollowingTransaction>
+                    <value>308564</value>
+                </sharesOwnedFollowingTransaction>
+            </postTransactionAmounts>
+            <ownershipNature>
+                <directOrIndirectOwnership>
+                    <value>D</value>
+                </directOrIndirectOwnership>
+            </ownershipNature>
+        </nonDerivativeTransaction>
+    </nonDerivativeTable>
+        """
+        data = self.parser.process(doc)
+        assert data == {
+            "nonDerivativeTable": {
+                "nonDerivativeTransaction": [
+                    {
+                        "securityTitle": "Common Stock",
+                        "transactionDate": "2021-05-14",
+                        "transactionCoding": {
+                            "transactionFormType": "5",
+                            "transactionCode": "G",
+                            "equitySwapInvolved": "0"
+                        },
+                        "transactionAmounts": {
+                            "transactionShares": "4010",
+                            "transactionPricePerShare": "0",
+                            "transactionAcquiredDisposedCode": "D"
+                        },
+                        "postTransactionAmounts": {
+                            "sharesOwnedFollowingTransaction": "324164"
+                        },
+                        "ownershipNature": {
+                            "directOrIndirectOwnership": "D"
+                        }
+                    },
+                    {
+                        "securityTitle": "Common Stock",
+                        "transactionDate": "2021-08-02",
+                        "transactionCoding": {
+                            "transactionFormType": "4",
+                            "transactionCode": "S",
+                            "equitySwapInvolved": "0"
+                        },
+                        "transactionAmounts": {
+                            "transactionShares": "15600",
+                            "transactionPricePerShare": "145.83",
+                            "transactionAcquiredDisposedCode": "D"
+                        },
+                        "postTransactionAmounts": {
+                            "sharesOwnedFollowingTransaction": "308564"
+                        },
+                        "ownershipNature": {
+                            "directOrIndirectOwnership": "D"
+                        }
+                    }
+                ]
+            }
+        }