Repository: salesforce/aws-allowlister (language: python)

Issue:
Change database creation flow
Right now, we have this process:

1. Populate Scraping Data Table. As the website is scraped, stash it in this table
2. Transform Scraping Data Table in place: we transform the data in place based on the overrides criteria before it is placed in this table.
3. Populate the Compliance Table based on the updated Scraping Data Table

We should create 3 tables instead of two. 
1. Raw Scraping Data Table: as the website is scraped, place the values in here.
2. Transformed Scraping Data Table: Transformed data based on overrides etc.
3. Compliance Table: The final data based on the updated scraping data table

This way, people can troubleshoot the quality from the SQLite Database directly without having to troubleshoot in an IDE.


Discussion / hints:

Fix patch:
diff --git a/aws_allowlister/command/rebuild.py b/aws_allowlister/command/rebuild.py
index 0806dfd..5a74321 100644
--- a/aws_allowlister/command/rebuild.py
+++ b/aws_allowlister/command/rebuild.py
@@ -1,15 +1,16 @@
 import click
-from aws_allowlister.database.build import build_compliance_database
+from aws_allowlister.database.build import build_database
 from aws_allowlister.database.database import connect_db
 from aws_allowlister.database.compliance_data import (
     ComplianceData,
-    update_compliance_database,
+    # update_compliance_database,
 )
 
 
 @click.command(name="rebuild", short_help="Rebuild the database")
 def rebuild():
     db_session = connect_db()
-    build_compliance_database()
-    update_compliance_database(db_session=db_session)
+    build_database()
     compliance_data = ComplianceData()
+    compliance_data.update_compliance_database(db_session=db_session)
+    # compliance_data = ComplianceData()
diff --git a/aws_allowlister/data/compliance.db b/aws_allowlister/data/compliance.db
index d254d48..25f0a55 100644
Binary files a/aws_allowlister/data/compliance.db and b/aws_allowlister/data/compliance.db differ
diff --git a/aws_allowlister/database/build.py b/aws_allowlister/database/build.py
index 8d75dfa..37e14ec 100644
--- a/aws_allowlister/database/build.py
+++ b/aws_allowlister/database/build.py
@@ -35,65 +35,67 @@ def create_empty_compliance_database(db_session):
         db_session.commit()
 
 
-def build_compliance_database():
+def build_database():
+    # Remove previous database file and connect
     if os.path.exists(DATABASE_PATH):
         os.remove(DATABASE_PATH)
     db_session = connect_db()
-    link = "https://aws.amazon.com/compliance/services-in-scope/"
-    response = requests.get(link, allow_redirects=False)
-    html_docs_destination = os.path.join(
-        os.path.dirname(__file__), os.path.pardir, "data"
-    )
-    file_name = "services-in-scope.html"
-    html_file_path = os.path.join(html_docs_destination, file_name)
-    if os.path.exists(html_file_path):
-        os.remove(html_file_path)
-
-    get_aws_html(link, html_docs_destination, file_name)
-
-    def get_standard_names(this_soup):
-        all_standard_names = []
-        for li in this_soup.find_all("li"):
-            if li.get("id"):
-                if li.get("id").startswith("aws-element"):
-                    all_standard_names.append(li.contents[1].text)
-        return all_standard_names
-
-    def get_table_ids(this_soup):
-        table_ids = []
-        for li in this_soup.find_all("li"):
-            if li.get("id"):
-                if li.get("id").startswith("aws-element"):
-                    table_ids.append(li.get("id"))
-        return table_ids
-
-    with open(os.path.join(html_docs_destination, file_name), "r") as f:
-
-        soup = BeautifulSoup(response.content, "html.parser")
-
-        standard_names = get_standard_names(this_soup=soup)
-        table_ids = get_table_ids(this_soup=soup)
-
-        create_empty_compliance_database(db_session)
-
-        these_results = []
-        for this_table_id in table_ids:
-            table = soup.find(id=this_table_id)
-
-            # Get the standard name based on the "tab" name
-            tab = table.contents[1]
-            standard_name = tab.contents[0]
-
-            # Skip certain cases based on inconsistent formatting
-            exclusions = ["FedRAMP", "DoD CC SRG", "HIPAA BAA", "MTCS"]
-            if standard_name in exclusions:
-                continue
-            rows = table.find_all("tr")
-            if len(rows) == 0:
-                continue
-
-            # Scrape it
-            result = scrape_standard_table(db_session, soup, this_table_id)
-            these_results.append(result)
+    # Start scraping the standard table
+    # link = "https://aws.amazon.com/compliance/services-in-scope/"
+    # response = requests.get(link, allow_redirects=False)
+    # html_docs_destination = os.path.join(
+    #     os.path.dirname(__file__), os.path.pardir, "data"
+    # )
+    # file_name = "services-in-scope.html"
+    # html_file_path = os.path.join(html_docs_destination, file_name)
+    # if os.path.exists(html_file_path):
+    #     os.remove(html_file_path)
+    #
+    # get_aws_html(link, html_docs_destination, file_name)
+
+    # def get_standard_names(this_soup):
+    #     all_standard_names = []
+    #     for li in this_soup.find_all("li"):
+    #         if li.get("id"):
+    #             if li.get("id").startswith("aws-element"):
+    #                 all_standard_names.append(li.contents[1].text)
+    #     return all_standard_names
+    #
+    # def get_table_ids(this_soup):
+    #     table_ids = []
+    #     for li in this_soup.find_all("li"):
+    #         if li.get("id"):
+    #             if li.get("id").startswith("aws-element"):
+    #                 table_ids.append(li.get("id"))
+    #     return table_ids
+
+    create_empty_compliance_database(db_session)
+    scrape_standard_table(db_session)
+    # with open(os.path.join(html_docs_destination, file_name), "r") as f:
+    #
+    #     soup = BeautifulSoup(response.content, "html.parser")
+    #
+    #     standard_names = get_standard_names(this_soup=soup)
+    #     table_ids = get_table_ids(this_soup=soup)
+    #
+    #     these_results = []
+    #     for this_table_id in table_ids:
+    #         table = soup.find(id=this_table_id)
+    #
+    #         # Get the standard name based on the "tab" name
+    #         tab = table.contents[1]
+    #         standard_name = tab.contents[0]
+    #
+    #         # Skip certain cases based on inconsistent formatting
+    #         exclusions = ["FedRAMP", "DoD CC SRG", "HIPAA BAA", "MTCS"]
+    #         if standard_name in exclusions:
+    #             continue
+    #         rows = table.find_all("tr")
+    #         if len(rows) == 0:
+    #             continue
+    #
+    #         # Scrape it
+    #         result = scrape_standard_table(db_session, soup, this_table_id)
+    #         these_results.append(result)
     scrape_iso_table(db_session)
     scrape_hipaa_table(db_session)
diff --git a/aws_allowlister/database/compliance_data.py b/aws_allowlister/database/compliance_data.py
index ea460ef..3f6400a 100644
--- a/aws_allowlister/database/compliance_data.py
+++ b/aws_allowlister/database/compliance_data.py
@@ -1,7 +1,8 @@
 from policy_sentry.shared.iam_data import get_service_prefix_data
 from policy_sentry.querying.all import get_all_service_prefixes
-from aws_allowlister.database.database import ComplianceTable, ScrapingDataTable
-from aws_allowlister.database.scraping_data import ScrapingData
+from aws_allowlister.database.database import ComplianceTable, TransformedScrapingDataTable, RawScrapingDataTable
+# from aws_allowlister.database.scraping_data import ScrapingData
+from aws_allowlister.database.transformed_scraping_data import TransformedScrapingData
 from aws_allowlister.scrapers.overrides import Overrides
 
 ALL_SERVICE_PREFIXES = get_all_service_prefixes()
@@ -67,65 +68,137 @@ def get_compliant_services(self, db_session, compliance_standard):
         db_session.close()
         return compliant_services
 
+    def update_database_by_matching_sdk_names_with_iam_prefixes(self, db_session, transformed_scraping_database):
+        for standard in self.standard_names(db_session):
+            sdk_names = transformed_scraping_database.get_sdk_names_matching_compliance_standard(
+                db_session, standard
+            )
+            # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+            # print("SDK names:")
+            # print(list(sdk_names.keys()))
 
-def update_compliance_database(db_session):
-    # get a list of all unique compliance standards in the ScrapingDataTable
-    query = db_session.query(
-        ScrapingDataTable.compliance_standard_name.distinct().label(
-            "compliance_standard_name"
-        )
-    )
-    standards = [row.compliance_standard_name for row in query.all()]
-    compliance = ComplianceData()
-    scraping_database = ScrapingData()
+            all_service_prefixes = get_all_service_prefixes()
+            for service_prefix in all_service_prefixes:
+                # If it matches the service prefix, update the compliance table (which has it sorted by IAM names) to check the box
+                if service_prefix in list(sdk_names.keys()):
+                    self.set_compliance_status(
+                        db_session=db_session,
+                        service_prefix=service_prefix,
+                        compliance_standard=standard,
+                        status="true",
+                    )
+                    # TODO: Should we also update the alternate names?
 
-    def update_database_by_matching_sdk_names_with_iam_prefixes(standard):
-        sdk_names = scraping_database.get_sdk_names_matching_compliance_standard(
-            db_session, standard
-        )
-        # Get a list of all SDK names matching this compliance standard from the ScrapingDataTable
-        # print("SDK names:")
-        # print(list(sdk_names.keys()))
+    def update_database_by_matching_compliance_names_with_iam_names(self, db_session, transformed_scraping_database):
+        for standard in self.standard_names(db_session):
+
+            # The service name in IAM-land
+            iam_service_names = {}
+            for service_prefix in ALL_SERVICE_PREFIXES:
+                iam_service_names[service_prefix] = get_service_prefix_data(service_prefix)[
+                    "service_name"
+                ]
 
-        all_service_prefixes = get_all_service_prefixes()
-        for service_prefix in all_service_prefixes:
-            # If it matches the service prefix, update the compliance table (which has it sorted by IAM names) to check the box
-            if service_prefix in list(sdk_names.keys()):
-                compliance.set_compliance_status(
-                    db_session=db_session,
-                    service_prefix=service_prefix,
-                    compliance_standard=standard,
-                    status="true",
+            # The service name in compliance land
+            compliance_service_names = (
+                transformed_scraping_database.get_service_names_matching_compliance_standard(
+                    db_session, standard
                 )
-                # TODO: Should we also update the alternate names?
+            )
+            for iam_service_prefix in list(iam_service_names.keys()):
+                iam_name = iam_service_names[iam_service_prefix]
+                compliance_names = list(compliance_service_names.keys())
+                if iam_name in compliance_names:
+                    self.set_compliance_status(
+                        db_session=db_session,
+                        service_prefix=iam_service_prefix,  # this is the IAM prefix
+                        compliance_standard=standard,
+                        status="true",
+                    )
 
-    def update_database_by_matching_compliance_names_with_iam_names(standard):
-        # The service name in IAM-land
-        iam_service_names = {}
-        for service_prefix in ALL_SERVICE_PREFIXES:
-            iam_service_names[service_prefix] = get_service_prefix_data(service_prefix)[
-                "service_name"
-            ]
+    def update_compliance_database(self, db_session, overrides=None):
+        if not overrides:
+            overrides = Overrides()
 
-        # The service name in compliance land
-        compliance_service_names = (
-            scraping_database.get_service_names_matching_compliance_standard(
-                db_session, standard
-            )
+        # get a list of all unique compliance standards in the TransformedScrapingDataTable
+        # query = db_session.query(
+        #     TransformedScrapingDataTable.compliance_standard_name.distinct().label(
+        #         "compliance_standard_name"
+        #     )
+        # )
+        # standards = [row.compliance_standard_name for row in query.all()]
+        # compliance = ComplianceData()
+        transformed_scraping_database = TransformedScrapingData()
+        transformed_scraping_database.populate_table(db_session, overrides)
+        # overrides = Overrides()
+        transformed_scraping_database.apply_overrides(db_session=db_session, overrides=overrides)
+        self.update_database_by_matching_sdk_names_with_iam_prefixes(
+            db_session=db_session, transformed_scraping_database=transformed_scraping_database
+        )
+        self.update_database_by_matching_compliance_names_with_iam_names(
+            db_session=db_session, transformed_scraping_database=transformed_scraping_database
         )
-        for iam_service_prefix in list(iam_service_names.keys()):
-            iam_name = iam_service_names[iam_service_prefix]
-            compliance_names = list(compliance_service_names.keys())
-            if iam_name in compliance_names:
-                compliance.set_compliance_status(
-                    db_session=db_session,
-                    service_prefix=iam_service_prefix,  # this is the IAM prefix
-                    compliance_standard=standard,
-                    status="true",
-                )
 
-    overrides = Overrides()
-    scraping_database.apply_overrides(db_session=db_session, overrides=overrides)
-    for some_standard in standards:
-        update_database_by_matching_sdk_names_with_iam_prefixes(some_standard)
-        update_database_by_matching_compliance_names_with_iam_names(some_standard)
+#
+# def update_compliance_database(db_session):
+#     # get a list of all unique compliance standards in the TransformedScrapingDataTable
+#     query = db_session.query(
+#         TransformedScrapingDataTable.compliance_standard_name.distinct().label(
+#             "compliance_standard_name"
+#         )
+#     )
+#     standards = [row.compliance_standard_name for row in query.all()]
+#     compliance = ComplianceData()
+#     transformed_scraping_database = TransformedScrapingData()
+#
+#     def update_database_by_matching_sdk_names_with_iam_prefixes(standard):
+#         sdk_names = transformed_scraping_database.get_sdk_names_matching_compliance_standard(
+#             db_session, standard
+#         )
+#         # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+#         # print("SDK names:")
+#         # print(list(sdk_names.keys()))
+#
+#         all_service_prefixes = get_all_service_prefixes()
+#         for service_prefix in all_service_prefixes:
+#             # If it matches the service prefix, update the compliance table (which has it sorted by IAM names) to check the box
+#             if service_prefix in list(sdk_names.keys()):
+#                 compliance.set_compliance_status(
+#                     db_session=db_session,
+#                     service_prefix=service_prefix,
+#                     compliance_standard=standard,
+#                     status="true",
+#                 )
+#                 # TODO: Should we also update the alternate names?
+#
+#     def update_database_by_matching_compliance_names_with_iam_names(standard):
+#         # The service name in IAM-land
+#         iam_service_names = {}
+#         for service_prefix in ALL_SERVICE_PREFIXES:
+#             iam_service_names[service_prefix] = get_service_prefix_data(service_prefix)[
+#                 "service_name"
+#             ]
+#
+#         # The service name in compliance land
+#         compliance_service_names = (
+#             transformed_scraping_database.get_service_names_matching_compliance_standard(
+#                 db_session, standard
+#             )
+#         )
+#         for iam_service_prefix in list(iam_service_names.keys()):
+#             iam_name = iam_service_names[iam_service_prefix]
+#             compliance_names = list(compliance_service_names.keys())
+#             if iam_name in compliance_names:
+#                 compliance.set_compliance_status(
+#                     db_session=db_session,
+#                     service_prefix=iam_service_prefix,  # this is the IAM prefix
+#                     compliance_standard=standard,
+#                     status="true",
+#                 )
+#
+#     overrides = Overrides()
+#     transformed_scraping_database.populate_table(db_session=db_session, overrides=overrides)
+#     # transformed_scraping_database.apply_overrides(db_session=db_session, overrides=overrides)
+#     for some_standard in standards:
+#         update_database_by_matching_sdk_names_with_iam_prefixes(some_standard)
+#         update_database_by_matching_compliance_names_with_iam_names(some_standard)
diff --git a/aws_allowlister/database/database.py b/aws_allowlister/database/database.py
index e3396b8..45eab6d 100644
--- a/aws_allowlister/database/database.py
+++ b/aws_allowlister/database/database.py
@@ -32,6 +32,40 @@ def connect_db():
     return db_session
 
 
+class RawScrapingDataTable(Base):
+    """The table that contains the compliance data, as presented by the 'Services in Scope' documentation"""
+
+    __tablename__ = "rawscrapingdatatable"
+    id = Column(Integer, primary_key=True)
+    sdk_name = Column(String(50))
+    service_name = Column(String(50))
+    compliance_standard_name = Column(String(50))
+
+    def __repr__(self):
+        return (
+            f"<RawScrapingDataTable(compliance_standard_name={self.compliance_standard_name}, "
+            f"sdk_name={self.sdk_name}, "
+            f"service_name='{self.service_name}')>"
+        )
+
+
+class TransformedScrapingDataTable(Base):
+    """The table that contains the compliance data, as presented by the 'Services in Scope' documentation"""
+
+    __tablename__ = "transformedscrapingdatatable"
+    id = Column(Integer, primary_key=True)
+    sdk_name = Column(String(50))
+    service_name = Column(String(50))
+    compliance_standard_name = Column(String(50))
+
+    def __repr__(self):
+        return (
+            f"<TransformedScrapingDataTable(compliance_standard_name={self.compliance_standard_name}, "
+            f"sdk_name={self.sdk_name}, "
+            f"service_name='{self.service_name}')>"
+        )
+
+
 class ComplianceTable(Base):
     """The table that you query for Compliance status using IAM names"""
 
@@ -65,20 +99,3 @@ def __repr__(self):
             f"OSPAR='{self.OSPAR}', "
             f"FINMA='{self.FINMA}')>"
         )
-
-
-class ScrapingDataTable(Base):
-    """The table that contains the compliance data, as presented by the 'Services in Scope' documentation"""
-
-    __tablename__ = "scrapingdatatable"
-    id = Column(Integer, primary_key=True)
-    sdk_name = Column(String(50))
-    service_name = Column(String(50))
-    compliance_standard_name = Column(String(50))
-
-    def __repr__(self):
-        return (
-            f"<ScrapingDataTable(compliance_standard_name={self.compliance_standard_name}, "
-            f"sdk_name={self.sdk_name}, "
-            f"service_name='{self.service_name}')>"
-        )
diff --git a/aws_allowlister/database/raw_scraping_data.py b/aws_allowlister/database/raw_scraping_data.py
new file mode 100644
index 0000000..86192e2
--- /dev/null
+++ b/aws_allowlister/database/raw_scraping_data.py
@@ -0,0 +1,47 @@
+import logging
+from aws_allowlister.database.database import RawScrapingDataTable
+logger = logging.getLogger(__name__)
+
+
+class RawScrapingData:
+    def __init__(self):
+        self.note = None
+
+    def standards(self, db_session):
+        query = db_session.query(
+            RawScrapingDataTable.compliance_standard_name.distinct().label(
+                "compliance_standard_name"
+            )
+        )
+        standards = [row.compliance_standard_name for row in query.all()]
+        return standards
+
+    def get_sdk_names_matching_compliance_standard(self, db_session, standard_name):
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(RawScrapingDataTable).filter(
+            RawScrapingDataTable.compliance_standard_name == standard_name
+        )
+        sdk_names = {}
+        for row in rows:
+            sdk_names[row.sdk_name] = row.service_name
+        return sdk_names
+
+    def get_service_names_matching_compliance_standard(self, db_session, standard_name):
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(RawScrapingDataTable).filter(
+            RawScrapingDataTable.compliance_standard_name == standard_name
+        )
+        service_names = {}
+        for row in rows:
+            service_names[row.service_name] = row.sdk_name
+        return service_names
+
+    def add_entry_to_database(self, db_session, compliance_standard_name, service_name, sdk):
+        db_session.add(
+            RawScrapingDataTable(
+                compliance_standard_name=compliance_standard_name,
+                sdk_name=sdk,
+                service_name=service_name,
+            )
+        )
+        db_session.commit()
diff --git a/aws_allowlister/database/scraping_data.py b/aws_allowlister/database/scraping_data.py
index 4bc8cfa..0c7f36c 100644
--- a/aws_allowlister/database/scraping_data.py
+++ b/aws_allowlister/database/scraping_data.py
@@ -1,5 +1,5 @@
 import logging
-from aws_allowlister.database.database import ScrapingDataTable
+from aws_allowlister.database.database import TransformedScrapingDataTable
 from aws_allowlister.scrapers.overrides import Overrides
 from aws_allowlister.shared.utils import get_service_name_matching_iam_service_prefix
 
@@ -12,7 +12,7 @@ def __init__(self):
 
     def standards(self, db_session):
         query = db_session.query(
-            ScrapingDataTable.compliance_standard_name.distinct().label(
+            TransformedScrapingDataTable.compliance_standard_name.distinct().label(
                 "compliance_standard_name"
             )
         )
@@ -20,9 +20,9 @@ def standards(self, db_session):
         return standards
 
     def get_sdk_names_matching_compliance_standard(self, db_session, standard_name):
-        # Get a list of all SDK names matching this compliance standard from the ScrapingDataTable
-        rows = db_session.query(ScrapingDataTable).filter(
-            ScrapingDataTable.compliance_standard_name == standard_name
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.compliance_standard_name == standard_name
         )
         sdk_names = {}
         for row in rows:
@@ -30,9 +30,9 @@ def get_sdk_names_matching_compliance_standard(self, db_session, standard_name):
         return sdk_names
 
     def get_service_names_matching_compliance_standard(self, db_session, standard_name):
-        # Get a list of all SDK names matching this compliance standard from the ScrapingDataTable
-        rows = db_session.query(ScrapingDataTable).filter(
-            ScrapingDataTable.compliance_standard_name == standard_name
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.compliance_standard_name == standard_name
         )
         service_names = {}
         for row in rows:
@@ -40,16 +40,16 @@ def get_service_names_matching_compliance_standard(self, db_session, standard_na
         return service_names
 
     def set_sdk_name_given_service_name(self, db_session, service_name, sdk_name):
-        rows = db_session.query(ScrapingDataTable).filter(
-            ScrapingDataTable.service_name == service_name
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.service_name == service_name
         )
         rows.update({"sdk_name": sdk_name})
         db_session.commit()
         db_session.close()
 
     def set_service_name_given_sdk_name(self, db_session, service_name, sdk_name):
-        rows = db_session.query(ScrapingDataTable).filter(
-            ScrapingDataTable.sdk_name == sdk_name
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.sdk_name == sdk_name
         )
         rows.update({"service_name": service_name})
         db_session.commit()
@@ -74,8 +74,8 @@ def override_service_names_to_iam_names(self, db_session, overrides):
         staging_area = {}
         content_to_remove = []
         for service_name in overrides.service_names_to_iam_names:
-            rows = db_session.query(ScrapingDataTable).filter(
-                ScrapingDataTable.service_name == service_name
+            rows = db_session.query(TransformedScrapingDataTable).filter(
+                TransformedScrapingDataTable.service_name == service_name
             )
             print(f"\n{service_name}: {rows.count()}")
             matching_standards = []
@@ -108,7 +108,7 @@ def override_service_names_to_iam_names(self, db_session, overrides):
         print("\nRemoving the old content:")
         for old_content in content_to_remove:
             print(f"\t{old_content}")
-            row = db_session.query(ScrapingDataTable).filter_by(
+            row = db_session.query(TransformedScrapingDataTable).filter_by(
                 compliance_standard_name=old_content.get("compliance_standard_name"),
                 service_name=old_content.get("service_name"),
                 sdk_name=old_content.get("sdk_name"),
@@ -125,7 +125,7 @@ def override_service_names_to_iam_names(self, db_session, overrides):
                 # for entry in staging_area.get(service_name).get(standard):
                 print(f"\t{entry}")
                 exists = (
-                    db_session.query(ScrapingDataTable)
+                    db_session.query(TransformedScrapingDataTable)
                     .filter_by(
                         compliance_standard_name=entry.get("compliance_standard_name"),
                         service_name=entry.get("service_name"),
@@ -134,7 +134,7 @@ def override_service_names_to_iam_names(self, db_session, overrides):
                     .first()
                 )
                 if not exists:
-                    new_entry = ScrapingDataTable(
+                    new_entry = TransformedScrapingDataTable(
                         compliance_standard_name=entry.get("compliance_standard_name"),
                         service_name=entry.get("service_name"),
                         sdk_name=entry.get("sdk_name"),
@@ -149,8 +149,8 @@ def override_sdk_names_to_iam_names(self, db_session, overrides):
         staging_area = {}
         content_to_remove = []
         for sdk_name in overrides.sdk_names_to_iam_names:
-            rows = db_session.query(ScrapingDataTable).filter(
-                ScrapingDataTable.sdk_name == sdk_name
+            rows = db_session.query(TransformedScrapingDataTable).filter(
+                TransformedScrapingDataTable.sdk_name == sdk_name
             )
             print(f"\n{sdk_name}: {rows.count()}")
             matching_standards = []
@@ -184,7 +184,7 @@ def override_sdk_names_to_iam_names(self, db_session, overrides):
         print("\nRemoving the old content:")
         for old_content in content_to_remove:
             print(f"\t{old_content}")
-            row = db_session.query(ScrapingDataTable).filter_by(
+            row = db_session.query(TransformedScrapingDataTable).filter_by(
                 compliance_standard_name=old_content.get("compliance_standard_name"),
                 service_name=old_content.get("service_name"),
                 sdk_name=old_content.get("sdk_name"),
@@ -201,7 +201,7 @@ def override_sdk_names_to_iam_names(self, db_session, overrides):
                 # for entry in staging_area.get(iam_name).get(standard):
                 print(f"\t{entry}")
                 exists = (
-                    db_session.query(ScrapingDataTable)
+                    db_session.query(TransformedScrapingDataTable)
                     .filter_by(
                         compliance_standard_name=entry.get("compliance_standard_name"),
                         service_name=entry.get("service_name"),
@@ -210,7 +210,7 @@ def override_sdk_names_to_iam_names(self, db_session, overrides):
                     .first()
                 )
                 if not exists:
-                    new_entry = ScrapingDataTable(
+                    new_entry = TransformedScrapingDataTable(
                         compliance_standard_name=entry.get("compliance_standard_name"),
                         service_name=entry.get("service_name"),
                         sdk_name=entry.get("sdk_name"),
@@ -227,7 +227,7 @@ def override_global_inserts(self, db_session, overrides):
         for standard in standards:
             for service_prefix in overrides.global_inserts:
                 exists = (
-                    db_session.query(ScrapingDataTable)
+                    db_session.query(TransformedScrapingDataTable)
                     .filter_by(
                         compliance_standard_name=standard, sdk_name=service_prefix
                     )
@@ -245,7 +245,7 @@ def override_global_inserts(self, db_session, overrides):
                         sdk_name=service_prefix,
                     )
                     print(f"\t{content}")
-                    new_entry = ScrapingDataTable(
+                    new_entry = TransformedScrapingDataTable(
                         compliance_standard_name=standard,
                         service_name=service_name,
                         sdk_name=service_prefix,
@@ -258,7 +258,7 @@ def add_scraping_entry_to_input_database(
     db_session, compliance_standard_name, service_name, sdk
 ):
     db_session.add(
-        ScrapingDataTable(
+        TransformedScrapingDataTable(
             compliance_standard_name=compliance_standard_name,
             sdk_name=sdk,
             service_name=service_name,
diff --git a/aws_allowlister/database/transformed_scraping_data.py b/aws_allowlister/database/transformed_scraping_data.py
new file mode 100644
index 0000000..cdcd5bc
--- /dev/null
+++ b/aws_allowlister/database/transformed_scraping_data.py
@@ -0,0 +1,293 @@
+import logging
+from aws_allowlister.database.database import TransformedScrapingDataTable, RawScrapingDataTable
+from aws_allowlister.scrapers.overrides import Overrides
+from aws_allowlister.shared.utils import get_service_name_matching_iam_service_prefix
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+
+class TransformedScrapingData:
+    def __init__(self):
+        self.note = None
+
+    def standards(self, db_session):
+        query = db_session.query(
+            TransformedScrapingDataTable.compliance_standard_name.distinct().label(
+                "compliance_standard_name"
+            )
+        )
+        standards = [row.compliance_standard_name for row in query.all()]
+        return standards
+
+    def get_sdk_names_matching_compliance_standard(self, db_session, standard_name):
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.compliance_standard_name == standard_name
+        )
+        sdk_names = {}
+        for row in rows:
+            sdk_names[row.sdk_name] = row.service_name
+        return sdk_names
+
+    def get_service_names_matching_compliance_standard(self, db_session, standard_name):
+        # Get a list of all SDK names matching this compliance standard from the TransformedScrapingDataTable
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.compliance_standard_name == standard_name
+        )
+        service_names = {}
+        for row in rows:
+            service_names[row.service_name] = row.sdk_name
+        return service_names
+
+    def set_sdk_name_given_service_name(self, db_session, service_name, sdk_name):
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.service_name == service_name
+        )
+        rows.update({"sdk_name": sdk_name})
+        db_session.commit()
+        db_session.close()
+
+    def set_service_name_given_sdk_name(self, db_session, service_name, sdk_name):
+        rows = db_session.query(TransformedScrapingDataTable).filter(
+            TransformedScrapingDataTable.sdk_name == sdk_name
+        )
+        rows.update({"service_name": service_name})
+        db_session.commit()
+        db_session.close()
+
+    def add_entry_to_database(self, db_session, compliance_standard_name, service_name, sdk_name):
+        db_session.add(
+            TransformedScrapingDataTable(
+                compliance_standard_name=compliance_standard_name,
+                sdk_name=sdk_name,
+                service_name=service_name,
+            )
+        )
+        db_session.commit()
+
+    def populate_table(self, db_session, overrides):
+        """Populate the Table using the initial Scraping Data, then apply the overrides"""
+        if not isinstance(overrides, Overrides):
+            raise Exception("Overrides should be an object class of type Overrides")
+
+        old_rows = db_session.query(RawScrapingDataTable)
+        # new_rows = db_session.query(TransformedScrapingDataTable)
+        logger.info("Populating the TransformedScrapingDataTable with the Raw Scraping Data initially")
+        for old_row in old_rows:
+            exists = db_session.query(TransformedScrapingDataTable).filter_by(
+                    compliance_standard_name=old_row.compliance_standard_name,
+                    service_name=old_row.service_name,
+                    sdk_name=old_row.sdk_name,
+                ).first()
+            if not exists:
+                self.add_entry_to_database(
+                    db_session=db_session,
+                    compliance_standard_name=old_row.compliance_standard_name,
+                    service_name=old_row.service_name,
+                    sdk_name=old_row.sdk_name,
+                )
+                # new_entry = TransformedScrapingDataTable(
+                # )
+                # db_session.add(new_entry)
+        # db_session.commit()
+        logger.info("Applying overrides to the TransformedScrapingDataTable")
+        self.apply_overrides(db_session=db_session, overrides=overrides)
+
+    def apply_overrides(self, db_session, overrides):
+        if not isinstance(overrides, Overrides):
+            raise Exception("Overrides should be an object class of type Overrides")
+        self.override_sdk_names_to_iam_names(db_session=db_session, overrides=overrides)
+        self.override_service_names_to_iam_names(
+            db_session=db_session, overrides=overrides
+        )
+        self.override_global_inserts(db_session=db_session, overrides=overrides)
+
+    def override_service_names_to_iam_names(self, db_session, overrides):
+        if not isinstance(overrides, Overrides):
+            raise Exception("Overrides should be an object class of type Overrides")
+        print("\nOVERRIDES: SECTION 1: override_service_names_to_iam_names")
+        print(
+            "\nApplying overrides to the database: Section 1, 'Service name means IAM name'"
+        )
+        staging_area = {}
+        content_to_remove = []
+        for service_name in overrides.service_names_to_iam_names:
+            rows = db_session.query(TransformedScrapingDataTable).filter(
+                TransformedScrapingDataTable.service_name == service_name
+            )
+            print(f"\n{service_name}: {rows.count()}")
+            matching_standards = []
+            print(f"\tCurrent content:")
+            for row in rows:
+                content = {
+                    "compliance_standard_name": row.compliance_standard_name,
+                    "service_name": row.service_name,
+                    "sdk_name": row.sdk_name,
+                }
+                print(f"\t{content}")
+                content_to_remove.append(content)
+                matching_standards.append(row.compliance_standard_name)
+            print(f"\t{service_name}'s matching standards: {matching_standards}")
+            for iam_name in overrides.service_names_to_iam_names.get(service_name):
+                if not staging_area.get(service_name):
+                    staging_area[service_name] = []
+                for matching_standard in matching_standards:
+                    # staging_area[service_name][matching_standard] = []
+                    content = {
+                        "compliance_standard_name": matching_standard,
+                        "service_name": service_name,
+                        "sdk_name": iam_name,
+                    }
+                    # print(f"\t{iam_name}")
+                    print(f"\tAdding to staging area: {content}")
+                    staging_area[service_name].append(content)
+                    # staging_area[service_name][matching_standard].append(content)
+        # Let's remove the old content since we are going to add the new stuff
+        print("\nRemoving the old content:")
+        for old_content in content_to_remove:
+            print(f"\t{old_content}")
+            row = db_session.query(TransformedScrapingDataTable).filter_by(
+                compliance_standard_name=old_content.get("compliance_standard_name"),
+                service_name=old_content.get("service_name"),
+                sdk_name=old_content.get("sdk_name"),
+            )
+            row.delete()
+            db_session.commit()
+
+        # Using the staging area to add to the database
+        print("\nAdding new content to the database")
+        for service_name in staging_area:
+            print(f"{service_name}: {len(staging_area.get(service_name))}")
+            print("\tNew content:")
+            for entry in staging_area.get(service_name):
+                # for entry in staging_area.get(service_name).get(standard):
+                print(f"\t{entry}")
+                exists = (
+                    db_session.query(TransformedScrapingDataTable)
+                    .filter_by(
+                        compliance_standard_name=entry.get("compliance_standard_name"),
+                        service_name=entry.get("service_name"),
+                        sdk_name=entry.get("sdk_name"),
+                    )
+                    .first()
+                )
+                if not exists:
+                    new_entry = TransformedScrapingDataTable(
+                        compliance_standard_name=entry.get("compliance_standard_name"),
+                        service_name=entry.get("service_name"),
+                        sdk_name=entry.get("sdk_name"),
+                    )
+                    db_session.add(new_entry)
+                    db_session.commit()
+
+    def override_sdk_names_to_iam_names(self, db_session, overrides):
+        if not isinstance(overrides, Overrides):
+            raise Exception("Overrides should be an object class of type Overrides")
+        print("\nOVERRIDES: SECTION 2: override_sdk_names_to_iam_names")
+        staging_area = {}
+        content_to_remove = []
+        for sdk_name in overrides.sdk_names_to_iam_names:
+            rows = db_session.query(TransformedScrapingDataTable).filter(
+                TransformedScrapingDataTable.sdk_name == sdk_name
+            )
+            print(f"\n{sdk_name}: {rows.count()}")
+            matching_standards = []
+            print(f"\tCurrent content:")
+            for row in rows:
+                content = {
+                    "compliance_standard_name": row.compliance_standard_name,
+                    "service_name": row.service_name,
+                    "sdk_name": row.sdk_name,
+                }
+                print(f"\t{content}")
+                content_to_remove.append(content)
+                matching_standards.append(row.compliance_standard_name)
+            print(f"\t{sdk_name}'s matching standards: {matching_standards}")
+            for iam_name in overrides.sdk_names_to_iam_names.get(sdk_name):
+                staging_area[iam_name] = []
+                for matching_standard in matching_standards:
+                    if not staging_area.get(iam_name):
+                        staging_area[iam_name] = []
+                    content = {
+                        "compliance_standard_name": matching_standard,
+                        "service_name": get_service_name_matching_iam_service_prefix(
+                            iam_name
+                        ),
+                        "sdk_name": iam_name,
+                    }
+                    # print(f"\t{iam_name}")
+                    print(f"\tAdding to staging area: {content}")
+                    staging_area[iam_name].append(content)
+        # Let's remove the old content since we are going to add the new stuff
+        print("\nRemoving the old content:")
+        for old_content in content_to_remove:
+            print(f"\t{old_content}")
+            row = db_session.query(TransformedScrapingDataTable).filter_by(
+                compliance_standard_name=old_content.get("compliance_standard_name"),
+                service_name=old_content.get("service_name"),
+                sdk_name=old_content.get("sdk_name"),
+            )
+            row.delete()
+            db_session.commit()
+
+        # Using the staging area to add to the database
+        print("\nAdding new content to the database")
+        for iam_name in staging_area:
+            print(f"{iam_name}: {len(staging_area.get(iam_name))}")
+            print("\tNew content:")
+            for entry in staging_area.get(iam_name):
+                # for entry in staging_area.get(iam_name).get(standard):
+                print(f"\t{entry}")
+                exists = (
+                    db_session.query(TransformedScrapingDataTable)
+                    .filter_by(
+                        compliance_standard_name=entry.get("compliance_standard_name"),
+                        service_name=entry.get("service_name"),
+                        sdk_name=entry.get("sdk_name"),
+                    )
+                    .first()
+                )
+                if not exists:
+                    new_entry = TransformedScrapingDataTable(
+                        compliance_standard_name=entry.get("compliance_standard_name"),
+                        service_name=entry.get("service_name"),
+                        sdk_name=entry.get("sdk_name"),
+                    )
+                    db_session.add(new_entry)
+                    db_session.commit()
+
+    def override_global_inserts(self, db_session, overrides):
+        if not isinstance(overrides, Overrides):
+            raise Exception("Overrides should be an object class of type Overrides")
+        standards = self.standards(db_session=db_session)
+        print("\nOVERRIDES: SECTION 3: override_global_inserts")
+        print("New content:")
+        for standard in standards:
+            for service_prefix in overrides.global_inserts:
+                exists = (
+                    db_session.query(TransformedScrapingDataTable)
+                    .filter_by(
+                        compliance_standard_name=standard, sdk_name=service_prefix
+                    )
+                    .first()
+                )
+                if not exists:
+                    service_name = get_service_name_matching_iam_service_prefix(
+                        service_prefix
+                    )
+                    if not service_name:
+                        service_name = service_prefix.capitalize()
+                    content = dict(
+                        compliance_standard_name=standard,
+                        service_name=service_name,
+                        sdk_name=service_prefix,
+                    )
+                    print(f"\t{content}")
+                    new_entry = TransformedScrapingDataTable(
+                        compliance_standard_name=standard,
+                        service_name=service_name,
+                        sdk_name=service_prefix,
+                    )
+                    db_session.add(new_entry)
+                    db_session.commit()
+
diff --git a/aws_allowlister/scrapers/aws_docs.py b/aws_allowlister/scrapers/aws_docs.py
index e3c90f7..8dadce8 100644
--- a/aws_allowlister/scrapers/aws_docs.py
+++ b/aws_allowlister/scrapers/aws_docs.py
@@ -6,9 +6,9 @@
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
+# TODO: Duplicate code passing in these parameters
 def get_aws_html(link, html_docs_destination, file_name):
     print(f"Getting the AWS documentation for: {file_name}")
-    # link = "https://aws.amazon.com/compliance/services-in-scope/"
     response = requests.get(link, allow_redirects=False)
     soup = BeautifulSoup(response.content, "html.parser")
 
@@ -47,7 +47,13 @@ def cleanup_links():
 
     cleanup_links()
 
-    with open(os.path.join(html_docs_destination, file_name), "w") as file:
+    file_path = os.path.join(html_docs_destination, file_name)
+    if os.path.exists(file_path):
+        print(f"Removing old file path: {file_path}")
+        os.remove(file_path)
+    with open(file_path, "w") as file:
+        print(f"Creating new file: {file_path}")
         file.write(str(soup.prettify()))
         file.close()
     logger.info("%s downloaded", file_name)
+    return file_path
diff --git a/aws_allowlister/scrapers/tables/hipaa.py b/aws_allowlister/scrapers/tables/hipaa.py
index f09aebf..291bbe1 100644
--- a/aws_allowlister/scrapers/tables/hipaa.py
+++ b/aws_allowlister/scrapers/tables/hipaa.py
@@ -3,7 +3,9 @@
 from bs4 import BeautifulSoup
 from policy_sentry.querying.all import get_all_service_prefixes
 from policy_sentry.shared.iam_data import get_service_prefix_data
-from aws_allowlister.database.scraping_data import ScrapingData, add_scraping_entry_to_input_database
+# from aws_allowlister.database.scraping_data import ScrapingData, add_scraping_entry_to_input_database
+from aws_allowlister.database.raw_scraping_data import RawScrapingData
+from aws_allowlister.database.transformed_scraping_data import TransformedScrapingData
 from aws_allowlister.scrapers.aws_docs import get_aws_html
 from aws_allowlister.shared.utils import clean_service_name
 
@@ -23,6 +25,7 @@ def scrape_hipaa_table(db_session):
     get_aws_html(link, html_docs_destination, file_name)
     response = requests.get(link, allow_redirects=False)
     soup = BeautifulSoup(response.content, "html.parser")
+    raw_scraping_data = RawScrapingData()
 
     service_names = []
 
@@ -41,17 +44,17 @@ def scrape_hipaa_table(db_session):
                 if tag.text not in false_positives:
                     service_names.append(tag.text)
     for service_name in service_names:
-        add_scraping_entry_to_input_database(
+        raw_scraping_data.add_entry_to_database(
             db_session=db_session,
             compliance_standard_name="HIPAA",
             sdk="",  # The HIPAA table does not list SDKs. We will update it to match in a second.
             service_name=clean_service_name(service_name),
         )
-    update_scraping_database_by_matching_hipaa_names_with_iam_names(db_session)
+    transform_database_by_matching_hipaa_names_with_iam_names(db_session)
 
 
-def update_scraping_database_by_matching_hipaa_names_with_iam_names(db_session):
-    scraping_database = ScrapingData()
+def transform_database_by_matching_hipaa_names_with_iam_names(db_session):
+    transformed_scraping_data = TransformedScrapingData()
     standard = "HIPAA"
     # The service name in IAM-land
     iam_service_names = {}
@@ -62,7 +65,7 @@ def update_scraping_database_by_matching_hipaa_names_with_iam_names(db_session):
 
     # The service name in compliance land
     compliance_service_names = (
-        scraping_database.get_service_names_matching_compliance_standard(
+        transformed_scraping_data.get_service_names_matching_compliance_standard(
             db_session, standard
         )
     )
@@ -71,7 +74,7 @@ def update_scraping_database_by_matching_hipaa_names_with_iam_names(db_session):
         compliance_names = list(compliance_service_names.keys())
         for name in compliance_names:
             if iam_name.lower() == name.lower():
-                scraping_database.set_sdk_name_given_service_name(
+                transformed_scraping_data.set_sdk_name_given_service_name(
                     db_session=db_session,
                     service_name=iam_name,
                     sdk_name=iam_service_prefix,
diff --git a/aws_allowlister/scrapers/tables/iso.py b/aws_allowlister/scrapers/tables/iso.py
index 5c344a6..edb95a2 100644
--- a/aws_allowlister/scrapers/tables/iso.py
+++ b/aws_allowlister/scrapers/tables/iso.py
@@ -2,7 +2,8 @@
 import requests
 from bs4 import BeautifulSoup, Tag
 from aws_allowlister.scrapers.aws_docs import get_aws_html
-from aws_allowlister.database.scraping_data import add_scraping_entry_to_input_database
+# from aws_allowlister.database.scraping_data import add_scraping_entry_to_input_database
+from aws_allowlister.database.raw_scraping_data import RawScrapingData
 from aws_allowlister.shared.utils import clean_service_name
 
 
@@ -16,11 +17,14 @@ def scrape_iso_table(db_session):
         os.remove(html_file_path)
     link = "https://aws.amazon.com/compliance/iso-certified/"
 
-    get_aws_html(link, html_docs_destination, file_name)
+    file_path = get_aws_html(link, html_docs_destination, file_name)
+
     response = requests.get(link, allow_redirects=False)
+    raw_scraping_data = RawScrapingData()
 
-    with open(os.path.join(html_docs_destination, file_name), "r") as f:
-        soup = BeautifulSoup(response.content, "html.parser")
+    with open(html_file_path, "r") as f:
+        soup = BeautifulSoup(f.read(), "html.parser")
+        # soup = BeautifulSoup(response.content, "html.parser")
         table = soup.find("tbody")
         rows = []
         for row in table.contents:
@@ -33,7 +37,7 @@ def scrape_iso_table(db_session):
                 sdk = str(row.contents[3].text)
                 if sdk == "Namespaces*" or service_name == "AWS Services":
                     continue
-                add_scraping_entry_to_input_database(
+                raw_scraping_data.add_entry_to_database(
                     db_session=db_session,
                     compliance_standard_name="ISO",
                     sdk=sdk,
diff --git a/aws_allowlister/scrapers/tables/standard.py b/aws_allowlister/scrapers/tables/standard.py
index 5e1b5ec..170b800 100644
--- a/aws_allowlister/scrapers/tables/standard.py
+++ b/aws_allowlister/scrapers/tables/standard.py
@@ -1,106 +1,173 @@
-from bs4 import Tag, NavigableString
+import os
+import requests
+from bs4 import Tag, NavigableString, BeautifulSoup
 from aws_allowlister.shared.utils import chomp, clean_service_name
-from aws_allowlister.database.scraping_data import add_scraping_entry_to_input_database
+# from aws_allowlister.database.scraping_data import add_scraping_entry_to_input_database
+from aws_allowlister.database.raw_scraping_data import RawScrapingData
+from aws_allowlister.scrapers.aws_docs import get_aws_html
 
 
-def scrape_standard_table(db_session, soup, table_id):
-    table = soup.find(id=table_id)
-    # Get the standard name based on the "tab" name
-    tab = table.contents[1]
-    standard_name = tab.contents[0]
-    # Get the rows and operate
-    rows = table.find_all("tr")
+# TODO: Move some of these cleanup items over to the Transform phase?
+#  At least the stuff where we remove special characters etc.
+def scrape_standard_table(db_session):
     results = []
 
-    def get_service_name(some_cells):
-        service_name_cell = some_cells[0].contents[0]
-        if isinstance(service_name_cell, NavigableString):
-            service_name = str(service_name_cell)
-        elif isinstance(service_name_cell, Tag):
-            service_name = service_name_cell.text
-        else:
-            service_name = str(service_name_cell)
-        service_name = clean_service_name(service_name)
-        # if translate_sdk_name_to_iam_prefix(service_prefix):
-        #     return True
-        return service_name
-
-    def clean_sdks(some_cells):
-        sdks = []
-        if len(some_cells) < 3:
+    # Get the file
+    html_docs_destination = os.path.join(
+        os.path.dirname(__file__), os.path.pardir, os.path.pardir, "data"
+    )
+    file_name = "services-in-scope.html"
+    html_file_path = os.path.join(html_docs_destination, file_name)
+    if os.path.exists(html_file_path):
+        os.remove(html_file_path)
+    # Start scraping the standard table
+    link = "https://aws.amazon.com/compliance/services-in-scope/"
+    file_path = get_aws_html(link, html_docs_destination, file_name)
+
+    raw_scraping_data = RawScrapingData()
+
+    with open(file_path, "r") as f:
+        soup = BeautifulSoup(f.read(), "html.parser")
+        # table = soup.find("tbody")
+        # table = soup.find(id=table_id)
+        # Get the standard name based on the "tab" name
+        # tab = table.contents[1]
+        # standard_name = tab.contents[0]
+        table_ids = get_table_ids(this_soup=soup)
+
+        # these_results = []
+        for this_table_id in table_ids:
+            table = soup.find(id=this_table_id)
+
+            # Get the standard name based on the "tab" name
+            tab = table.contents[1]
+            standard_name = chomp(str(tab.contents[0]))
+            print(f"Scraping table for {standard_name}")
+
+            # Skip certain cases based on inconsistent formatting
+            exclusions = ["FedRAMP", "DoD CC SRG", "HIPAA BAA", "MTCS"]
+            if standard_name in exclusions:
+                continue
+            rows = table.find_all("tr")
+            if len(rows) == 0:
+                continue
+
+            # Scrape it
+
+            for row in rows:
+                cells = row.find_all("td")
+                # Skip the first row, the rest are the same
+                if len(cells) == 0 or len(cells) == 1:
+                    continue
+
+                # Cell 0: Service name
+
+                this_service_name = get_service_name(cells)
+
+                # Cell 1: SDKs
+                # For the HIPAA BAA compliance standard, there are only two columns 🙄 smh at inconsistency
+                these_sdks = clean_sdks(cells)
+
+                # Cell 2: Status cell
+                # This will contain a checkmark (✓). Let's just mark as true if it is non-empty
+                this_status, this_status_cell_contents = clean_status_cell(cells)
+
+                result = dict(
+                    service_name=this_service_name,
+                    sdks=these_sdks,
+                    status=this_status,
+                    status_text=this_status_cell_contents,
+                )
+                for sdk in these_sdks:
+                    raw_scraping_data.add_entry_to_database(
+                        db_session=db_session,
+                        compliance_standard_name=standard_name,
+                        sdk=sdk,
+                        service_name=this_service_name,
+                    )
+                results.append(result)
+    return results
+
+
+def get_service_name(some_cells):
+    # try:
+    service_name_cell = some_cells[0].contents[1]
+    # except IndexError as i_e:
+    #     print(i_e)
+    #     service_name_cell = some_cells[0].contents[1]
+    if isinstance(service_name_cell, NavigableString):
+        service_name = str(service_name_cell)
+    elif isinstance(service_name_cell, Tag):
+        service_name = service_name_cell.text
+    else:
+        service_name = str(service_name_cell)
+    service_name = clean_service_name(service_name)
+    # if translate_sdk_name_to_iam_prefix(service_prefix):
+    #     return True
+    return service_name
+
+
+def clean_sdks(some_cells):
+    sdks = []
+    if len(some_cells) < 3:
+        pass
+    # Otherwise,
+    else:
+        cell_content = chomp(some_cells[1].contents[0])
+        if "<a" in cell_content or "]" in cell_content:
+            cell_content = some_cells[1].contents[0].text
+        if not cell_content:
             pass
-        # Otherwise,
         else:
-            cell_content = chomp(some_cells[1].contents[0])
-            if "<a" in cell_content or "]" in cell_content:
-                cell_content = some_cells[1].contents[0].text
-            if not cell_content:
-                pass
+            if "," in cell_content:
+                tmp_sdk_list = cell_content.split(",")
+                for tmp_sdk in tmp_sdk_list:
+                    sdks.append(chomp(tmp_sdk))
             else:
-                if "," in cell_content:
-                    tmp_sdk_list = cell_content.split(",")
-                    for tmp_sdk in tmp_sdk_list:
-                        sdks.append(chomp(tmp_sdk))
-                else:
-                    sdks = [cell_content]
-        return sdks
-
-    def clean_status_cell():
-        # Slice syntax in case there are only two columns
-        status_cell_contents = cells[-1].contents[0]
-
-        if status_cell_contents is None:
-            status = False
-        elif isinstance(status_cell_contents, str):
-            status_cell_contents = chomp(status_cell_contents)
-        elif isinstance(status_cell_contents, Tag):
-            status_cell_contents = chomp(status_cell_contents.text)
-        elif isinstance(status_cell_contents, NavigableString):
-            status_cell_contents = chomp(str(status_cell_contents))
-        else:
-            print("idk what type it is")
-
-        if status_cell_contents is None:
-            status = False
-        elif "✓" in status_cell_contents:
-            status = True
-        elif status_cell_contents != "✓":
-            status = False
-        else:
-            status = True
-
-        return status, status_cell_contents
-
-    for row in rows:
-        cells = row.find_all("td")
-        # Skip the first row, the rest are the same
-        if len(cells) == 0:
-            continue
-
-        # Cell 0: Service name
-
-        this_service_name = get_service_name(cells)
-
-        # Cell 1: SDKs
-        # For the HIPAA BAA compliance standard, there are only two columns 🙄 smh at inconsistency
-        these_sdks = clean_sdks(cells)
-
-        # Cell 2: Status cell
-        # This will contain a checkmark (✓). Let's just mark as true if it is non-empty
-        this_status, this_status_cell_contents = clean_status_cell()
-
-        result = dict(
-            service_name=this_service_name,
-            sdks=these_sdks,
-            status=this_status,
-            status_text=this_status_cell_contents,
-        )
-        for sdk in these_sdks:
-            add_scraping_entry_to_input_database(
-                db_session=db_session,
-                compliance_standard_name=standard_name,
-                sdk=sdk,
-                service_name=this_service_name,
-            )
-        results.append(result)
-    return results
+                sdks = [cell_content]
+    return sdks
+
+
+def clean_status_cell(cells):
+    # Slice syntax in case there are only two columns
+    status_cell_contents = cells[-1].contents[0]
+
+    if status_cell_contents is None:
+        status = False
+    elif isinstance(status_cell_contents, str):
+        status_cell_contents = chomp(status_cell_contents)
+    elif isinstance(status_cell_contents, Tag):
+        status_cell_contents = chomp(status_cell_contents.text)
+    elif isinstance(status_cell_contents, NavigableString):
+        status_cell_contents = chomp(str(status_cell_contents))
+    else:
+        print("idk what type it is")
+
+    if status_cell_contents is None:
+        status = False
+    elif "✓" in status_cell_contents:
+        status = True
+    elif status_cell_contents != "✓":
+        status = False
+    else:
+        status = True
+
+    return status, status_cell_contents
+
+
+def get_table_ids(this_soup):
+    table_ids = []
+    for li in this_soup.find_all("li"):
+        if li.get("id"):
+            if li.get("id").startswith("aws-element"):
+                table_ids.append(li.get("id"))
+    return table_ids
+
+
+def get_standard_names(this_soup):
+    all_standard_names = []
+    for li in this_soup.find_all("li"):
+        if li.get("id"):
+            if li.get("id").startswith("aws-element"):
+                all_standard_names.append(li.contents[1].text)
+    return all_standard_names
diff --git a/aws_allowlister/shared/utils.py b/aws_allowlister/shared/utils.py
index af917a4..0ace486 100644
--- a/aws_allowlister/shared/utils.py
+++ b/aws_allowlister/shared/utils.py
@@ -73,8 +73,13 @@ def clean_service_name(service_name):
 
     service_name, sep, tail = service_name.partition("[")
     service_name, sep, tail = service_name.partition("(")
-    service_name = re.sub("^[ ]*", "", service_name)  # Clean start
-    service_name = re.sub("[ ]*$", "", service_name)  # Clean end
+    # Remove tabs and newlines
+    service_name = service_name.replace('\n', '')
+    service_name = service_name.replace('\t', '')
+    # Clean start
+    service_name = re.sub("^[ ]*", "", service_name)
+    # Clean end
+    service_name = re.sub("[ ]*$", "", service_name)
     return service_name
 
 
diff --git a/utils/update_data.py b/utils/update_data.py
index 35fedbf..f846436 100644
--- a/utils/update_data.py
+++ b/utils/update_data.py
@@ -1,12 +1,12 @@
 from aws_allowlister.database.database import connect_db
-from aws_allowlister.database.build import build_compliance_database
+from aws_allowlister.database.build import build_database
 from aws_allowlister.database.compliance_data import update_compliance_database
 
 
 def update_data():
     db_session = connect_db()
-    build_compliance_database()
-    update_compliance_database(db_session=db_session)
+    build_database()
+    # update_compliance_database(db_session=db_session)
 
 
 if __name__ == '__main__':


Test patch:
diff --git a/test/test_db.py b/test/test_db.py
index 772dea2..4a95534 100644
--- a/test/test_db.py
+++ b/test/test_db.py
@@ -1,7 +1,7 @@
 import unittest
-from aws_allowlister.database.compliance_data import ComplianceData, ScrapingData
-from aws_allowlister.database.build import build_compliance_database
-from aws_allowlister.database.database import connect_db, ScrapingDataTable
+from aws_allowlister.database.compliance_data import ComplianceData
+from aws_allowlister.database.build import build_database
+from aws_allowlister.database.database import connect_db, TransformedScrapingDataTable
 from aws_allowlister.shared.utils import write_json_to_file
 from policy_sentry.shared.iam_data import get_service_prefix_data
 
diff --git a/test/test_utils.py b/test/test_utils.py
new file mode 100644
index 0000000..70881bf
--- /dev/null
+++ b/test/test_utils.py
@@ -0,0 +1,10 @@
+import unittest
+from aws_allowlister.shared.utils import clean_service_name
+
+
+class UtilsTestCase(unittest.TestCase):
+    def test_utils(self):
+        result = clean_service_name('\n\n\t\tAmazon API Gateway\t\n')
+        self.assertEqual(result, "Amazon API Gateway")
+        result = clean_service_name('Amazon API Gateway\n')
+        self.assertTrue(result == "Amazon API Gateway")