Repository: mtgjson/mtgjson

Issue:
Support Decks inside of Set Files & AllPrintings
It would be nice to have decklists embedded in the main AllPrintings data model for ease of access when determining ev and other properties. I see a couple of ways to do this

1. add the current decklists in the set data
2. add a fixed sheet in Booster, with custom Contents
3. add a separate data format for decks with just a (fixed) sheet

I'm favoring #2 because it would allow to reuse parsing format used for normal products, but I'd like to hear more thoughts - cc @ZeldaZach @taw @axxroytovu 

Discussion:
ZeldaZach: This still has room for improvement, as non-decks are in the decks array. Might need to filter on them

Patch:
diff --git a/mtgjson5/__main__.py b/mtgjson5/__main__.py
index 9b0d1da..3763a2c 100644
--- a/mtgjson5/__main__.py
+++ b/mtgjson5/__main__.py
@@ -95,7 +95,7 @@ def dispatcher(args: argparse.Namespace) -> None:
 
     # If a price build, simply build prices and exit
     if args.price_build:
-        generate_compiled_prices_output(build_prices(), args.pretty)
+        generate_compiled_prices_output(*build_prices(), args.pretty)
         if args.compress:
             compress_mtgjson_contents(MtgjsonConfig().output_path)
         generate_output_file_hashes(MtgjsonConfig().output_path)
diff --git a/mtgjson5/classes/__init__.py b/mtgjson5/classes/__init__.py
index cd41351..028af78 100644
--- a/mtgjson5/classes/__init__.py
+++ b/mtgjson5/classes/__init__.py
@@ -21,4 +21,5 @@ from .mtgjson_sealed_product import (
     MtgjsonSealedProductSubtype,
 )
 from .mtgjson_set import MtgjsonSetObject
+from .mtgjson_set_deck import MtgjsonSetDeckObject
 from .mtgjson_translations import MtgjsonTranslationsObject
diff --git a/mtgjson5/classes/mtgjson_sealed_product.py b/mtgjson5/classes/mtgjson_sealed_product.py
index 9b7e4bc..70c655e 100644
--- a/mtgjson5/classes/mtgjson_sealed_product.py
+++ b/mtgjson5/classes/mtgjson_sealed_product.py
@@ -156,3 +156,207 @@ class MtgjsonSealedProductObject:
             and key not in self.__skip_keys
             and value
         }
+
+    def determine_mtgjson_sealed_product_category(
+        self, product_name: str
+    ) -> Optional[MtgjsonSealedProductCategory]:
+        """
+        Best-effort to parse the product name and determine the sealed product category
+        :param product_name Name of the product from provider, must be lowercase
+        :return: category
+        """
+
+        # The order of the checks is important: for example, we need to catch all 'case'
+        # products before everything else, since "booster box case" would be caught in
+        # the 'booster box' check. The same applies to several sub products, such as
+        # 'prerelease' (vs guild kit), 'box set' (vs set boosters), and so on.
+        if any(
+            tag in product_name
+            for tag in [
+                "booster case",
+                "box case",
+                "bundle case",
+                "display case",
+                "intro display",
+                "intro pack display",
+                "pack box",
+                "pack case",
+                "tournament pack display",
+                "vip edition box",
+            ]
+        ):
+            return MtgjsonSealedProductCategory.CASE
+
+        if any(
+            tag in product_name
+            for tag in ["booster box", "booster display", "mythic edition"]
+        ):
+            return MtgjsonSealedProductCategory.BOOSTER_BOX
+
+        # Needs to be before BOOSTER_PACK due to aliasing
+        if any(
+            tag in product_name
+            for tag in [
+                "blister pack",
+                "booster draft pack",
+                "booster packs draft set",
+                "multipack",
+            ]
+        ):
+            return MtgjsonSealedProductCategory.DRAFT_SET
+
+        if any(
+            tag in product_name
+            for tag in [
+                "booster pack",
+                "booster retail pack",
+                "box topper",
+                "hanger pack",
+                "omega pack",
+                "promo pack",
+                "theme booster",
+                "vip edition pack",
+            ]
+        ):
+            if "set of" in product_name:
+                return MtgjsonSealedProductCategory.SUBSET
+            return MtgjsonSealedProductCategory.BOOSTER_PACK
+
+        if "prerelease" in product_name:
+            return MtgjsonSealedProductCategory.PRERELEASE_PACK
+
+        if any(
+            tag in product_name
+            for tag in [
+                "booster battle pack",
+                "clash pack",
+                "fourth edition gift box",
+                "quick start set",
+                "two player starter",
+            ]
+        ):
+            return MtgjsonSealedProductCategory.TWO_PLAYER_STARTER_SET
+
+        if any(
+            tag in product_name
+            for tag in [
+                "box set",
+                "builders toolkit",
+                "commander collection",
+                "deckmasters tin",
+                "deluxe collection",
+                "edition box",
+                "game night",
+                "global series",
+                "hascon",
+                "modern event deck",
+                "planechase",
+                "planeswalker set",
+                "premium deck series",
+                "sdcc",
+                "secret lair",
+                "signature spellbook",
+            ]
+        ):
+            # In this section, only Planechase may have a "set of"
+            if "planechase" in product_name and "set of" in product_name:
+                return MtgjsonSealedProductCategory.CASE
+            return MtgjsonSealedProductCategory.BOX_SET
+
+        if "commander" in product_name or "brawl deck" in product_name:
+            if "set of" in product_name:
+                return MtgjsonSealedProductCategory.CASE
+            return MtgjsonSealedProductCategory.COMMANDER_DECK
+
+        if "deck box" in product_name or "deck display" in product_name:
+            return MtgjsonSealedProductCategory.DECK_BOX
+
+        if any(
+            tag in product_name
+            for tag in [
+                "challenge deck",
+                "challenger deck",
+                "championship deck",
+                "event deck",
+                "guild kit",
+                "intro pack",
+                "planeswalker deck",
+                "starter deck",
+                "theme deck",
+                "tournament deck",
+                "tournament pack",
+            ]
+        ):
+            if "set of" in product_name:
+                return MtgjsonSealedProductCategory.SUBSET
+            return MtgjsonSealedProductCategory.DECK
+
+        if any(
+            tag in product_name
+            for tag in [
+                "bundle",
+                "fat pack",
+                "gift box",
+            ]
+        ):
+            return MtgjsonSealedProductCategory.BUNDLE
+
+        if "land station" in product_name:
+            return MtgjsonSealedProductCategory.LAND_STATION
+
+        return None
+
+    # Best-effort to parse the product name and determine the sealed product category
+    def determine_mtgjson_sealed_product_subtype(
+        self, product_name: str, category: Optional[MtgjsonSealedProductCategory]
+    ) -> Optional[MtgjsonSealedProductSubtype]:
+        """
+        Best-effort to parse the product name and determine the sealed product subtype
+        :param product_name Name of the product from provider
+        :param category Category as parsed from determine_mtgjson_sealed_product_category()
+        :return: subtype
+        """
+        if not category:
+            return None
+
+        for subtype in MtgjsonSealedProductSubtype:
+            if not subtype or not subtype.value:
+                continue
+
+            # Prevent aliasing from Eventide
+            if (
+                subtype is MtgjsonSealedProductSubtype.EVENT
+                and category is not MtgjsonSealedProductCategory.DECK
+            ):
+                continue
+
+            # Prevent assigning 'set' or 'draft_set' to Core Set editions
+            if "core set" in product_name and "set" in subtype.value:
+                continue
+
+            # Skip 'collector' for this set, since they weren't introduced it yet
+            if (
+                "ravnica allegiance" in product_name
+                and subtype is MtgjsonSealedProductSubtype.COLLECTOR
+            ):
+                continue
+
+            # Prevent assigning 'set' (for set boosters) to unrelated categories
+            if subtype is MtgjsonSealedProductSubtype.SET and (
+                category is not MtgjsonSealedProductCategory.BOOSTER_PACK
+                and category is not MtgjsonSealedProductCategory.BOOSTER_BOX
+            ):
+                continue
+
+            # Do the replace to use the tag as text
+            if subtype.value and subtype.value.replace("_", " ") in product_name:
+                return subtype
+
+        # Special handling because sometimes 'default' is not tagged
+        if category in [
+            MtgjsonSealedProductCategory.BOOSTER_BOX,
+            MtgjsonSealedProductCategory.BOOSTER_PACK,
+            MtgjsonSealedProductCategory.DRAFT_SET,
+        ]:
+            return MtgjsonSealedProductSubtype.DEFAULT
+        return None
diff --git a/mtgjson5/classes/mtgjson_set.py b/mtgjson5/classes/mtgjson_set.py
index 9ccc385..d47ff38 100644
--- a/mtgjson5/classes/mtgjson_set.py
+++ b/mtgjson5/classes/mtgjson_set.py
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Set
 from .. import constants
 from ..classes.mtgjson_card import MtgjsonCardObject
 from ..classes.mtgjson_sealed_product import MtgjsonSealedProductObject
+from ..classes.mtgjson_set_deck import MtgjsonSetDeckObject
 from ..classes.mtgjson_translations import MtgjsonTranslationsObject
 from ..utils import to_camel_case
 
@@ -22,6 +23,7 @@ class MtgjsonSetObject:
     cardsphere_set_id: Optional[int]
     code: str
     code_v3: str
+    decks: List[MtgjsonSetDeckObject]
     is_foreign_only: bool
     is_foil_only: bool
     is_non_foil_only: bool
diff --git a/mtgjson5/classes/mtgjson_set_deck.py b/mtgjson5/classes/mtgjson_set_deck.py
new file mode 100644
index 0000000..a3f75d7
--- /dev/null
+++ b/mtgjson5/classes/mtgjson_set_deck.py
@@ -0,0 +1,75 @@
+"""
+MTGJSON Set Deck Object
+"""
+import re
+from typing import Any, Dict, List, Optional
+
+from ..utils import to_camel_case
+from .mtgjson_sealed_product import MtgjsonSealedProductObject
+
+
+class MtgjsonSetDeckObject:
+    """
+    MTGJSON Set Deck Object
+    """
+
+    class MtgjsonSetDeckCardObject:
+        """
+        MTGJSON Set Deck Card Object
+        """
+
+        uuid: str
+        count: int
+        finish: str
+
+        def __init__(self, uuid: str, count: int, finish: str) -> None:
+            self.uuid = uuid
+            self.count = count
+            self.finish = finish
+
+        def to_json(self) -> Dict[str, Any]:
+            """
+            Support json.dump()
+            :return: JSON serialized object
+            """
+            return {
+                to_camel_case(key): value
+                for key, value in self.__dict__.items()
+                if "__" not in key and not callable(value)
+            }
+
+    name: str
+    sealed_product_uuids: Optional[List[str]]
+    cards: List[MtgjsonSetDeckCardObject]
+    __alpha_numeric_name: str
+
+    def __init__(self, name: str, sealed_product_uuids: Optional[List[str]]) -> None:
+        self.name = name
+        self.sealed_product_uuids = sealed_product_uuids
+        self.cards = []
+        self.__alpha_numeric_name = re.sub(r"[^A-Za-z0-9 ]+", "", self.name).lower()
+
+    def add_sealed_product_uuids(
+        self, mtgjson_set_sealed_products: List[MtgjsonSealedProductObject]
+    ) -> None:
+        """
+        Update the UUID for the deck to link back to sealed product, if able
+        :param mtgjson_set_sealed_products MTGJSON Set Sealed Products for this Set
+        """
+        if not self.sealed_product_uuids:
+            for sealed_product_entry in mtgjson_set_sealed_products:
+                sealed_name = sealed_product_entry.name.lower()
+                if self.__alpha_numeric_name in sealed_name:
+                    self.sealed_product_uuids = [sealed_product_entry.uuid]
+                    break
+
+    def to_json(self) -> Dict[str, Any]:
+        """
+        Support json.dump()
+        :return: JSON serialized object
+        """
+        return {
+            to_camel_case(key): value
+            for key, value in self.__dict__.items()
+            if "__" not in key and not callable(value)
+        }
diff --git a/mtgjson5/compiled_classes/mtgjson_structures.py b/mtgjson5/compiled_classes/mtgjson_structures.py
index f0321e5..1f95abb 100644
--- a/mtgjson5/compiled_classes/mtgjson_structures.py
+++ b/mtgjson5/compiled_classes/mtgjson_structures.py
@@ -56,6 +56,7 @@ class MtgjsonStructuresObject:
         self.all_printings = "AllPrintings"
         self.atomic_cards = "AtomicCards"
         self.all_prices = "AllPrices"
+        self.all_prices_today = "AllPricesToday"
         self.all_csvs_directory = "AllPrintingsCSVFiles"
         self.all_parquets_directory = "AllPrintingsParquetFiles"
         self.all_decks_directory = "AllDeckFiles"
diff --git a/mtgjson5/output_generator.py b/mtgjson5/output_generator.py
index e722df1..c0f32f6 100644
--- a/mtgjson5/output_generator.py
+++ b/mtgjson5/output_generator.py
@@ -30,16 +30,27 @@ LOGGER = logging.getLogger(__name__)
 
 
 def generate_compiled_prices_output(
-    price_data: Dict[str, Dict[str, float]], pretty_print: bool
+    all_price_data: Dict[str, Any], today_price_data: Dict[str, Any], pretty_print: bool
 ) -> None:
     """
     Dump AllPrices to a file
-    :param price_data: Data to dump
+    :param all_price_data: Data to dump for larger file
+    :param today_price_data: data to dump for smaller file
     :param pretty_print: Pretty or minimal
     """
     LOGGER.info("Building Prices")
     create_compiled_output(
-        MtgjsonStructuresObject().all_prices, price_data, pretty_print, sort_keys=False
+        MtgjsonStructuresObject().all_prices,
+        all_price_data,
+        pretty_print,
+        sort_keys=False,
+    )
+
+    create_compiled_output(
+        MtgjsonStructuresObject().all_prices_today,
+        today_price_data,
+        pretty_print,
+        sort_keys=False,
     )
 
 
@@ -187,7 +198,7 @@ def generate_compiled_output_files(pretty_print: bool) -> None:
     )
 
     # AllPrices.json
-    generate_compiled_prices_output(build_prices(), pretty_print)
+    generate_compiled_prices_output(*build_prices(), pretty_print)
 
     # CompiledList.json
     create_compiled_output(
diff --git a/mtgjson5/price_builder.py b/mtgjson5/price_builder.py
index 8e27baf..0085e14 100644
--- a/mtgjson5/price_builder.py
+++ b/mtgjson5/price_builder.py
@@ -7,7 +7,7 @@ import logging
 import lzma
 import pathlib
 import subprocess
-from typing import Any, Dict
+from typing import Any, Dict, Tuple
 
 import dateutil.relativedelta
 import mergedeep
@@ -180,7 +180,7 @@ def download_old_all_printings() -> None:
         f.write(lzma.decompress(file_bytes).decode())
 
 
-def build_prices() -> Dict[str, Any]:
+def build_prices() -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """
     The full build prices operation
     Prune & Update remote database
@@ -198,10 +198,10 @@ def build_prices() -> Dict[str, Any]:
     today_prices = build_today_prices()
     if not today_prices:
         LOGGER.warning("Pricing information failed to generate")
-        return {}
+        return {}, {}
 
     if not MtgjsonConfig().has_section("Prices"):
-        return today_prices
+        return today_prices, today_prices
 
     bucket_name = MtgjsonConfig().get("Prices", "bucket_name")
     bucket_object_path = MtgjsonConfig().get("Prices", "bucket_object_path")
@@ -211,7 +211,6 @@ def build_prices() -> Dict[str, Any]:
     # Update local copy of database
     LOGGER.info("Merging old and new price data")
     mergedeep.merge(archive_prices, today_prices)
-    del today_prices
 
     # Prune local copy of database
     LOGGER.info("Pruning price data")
@@ -226,4 +225,4 @@ def build_prices() -> Dict[str, Any]:
     MtgjsonS3Handler().upload_file(str(local_zip_file), bucket_name, bucket_object_path)
     local_zip_file.unlink()
 
-    return archive_prices
+    return archive_prices, today_prices
diff --git a/mtgjson5/providers/cardkingdom.py b/mtgjson5/providers/cardkingdom.py
index e1ea53e..04f6340 100644
--- a/mtgjson5/providers/cardkingdom.py
+++ b/mtgjson5/providers/cardkingdom.py
@@ -12,9 +12,7 @@ from singleton_decorator import singleton
 from .. import constants
 from ..classes import (
     MtgjsonPricesObject,
-    MtgjsonSealedProductCategory,
     MtgjsonSealedProductObject,
-    MtgjsonSealedProductSubtype,
 )
 from ..providers.abstract import AbstractProvider
 from ..utils import generate_card_mapping, retryable_session
@@ -106,197 +104,6 @@ class CardKingdomProvider(AbstractProvider):
             buy_key="price_buy",
         )
 
-    def determine_mtgjson_sealed_product_category(
-        self, product_name: str
-    ) -> Optional[MtgjsonSealedProductCategory]:
-        """
-        Best-effort to parse the product name and determine the sealed product category
-        :param product_name Name of the product from TCG, must be lowercase
-        :return: category
-        """
-
-        # The order of the checks is important: for example, we need to catch all 'case'
-        # products before everything else, since "booster box case" would be caught in
-        # the 'booster box' check. The same applies to several sub products, such as
-        # 'prerelease' (vs guild kit), 'box set' (vs set boosters), and so on.
-        if any(
-            tag in product_name
-            for tag in [
-                "booster case",
-                "box case",
-                "bundle case",
-                "display case",
-                "intro display",
-                "intro pack display",
-                "pack box",
-                "pack case",
-                "tournament pack display",
-                "vip edition box",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.CASE
-
-        if any(
-            tag in product_name
-            for tag in ["booster box", "booster display", "mythic edition"]
-        ):
-            return MtgjsonSealedProductCategory.BOOSTER_BOX
-
-        if any(
-            tag in product_name
-            for tag in [
-                "booster pack",
-                "booster retail pack",
-                "box topper",
-                "hanger pack",
-                "omega pack",
-                "promo pack",
-                "theme booster",
-                "vip edition pack",
-            ]
-        ):
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.SUBSET
-            return MtgjsonSealedProductCategory.BOOSTER_PACK
-
-        if "prerelease" in product_name:
-            return MtgjsonSealedProductCategory.PRERELEASE_PACK
-
-        if any(
-            tag in product_name
-            for tag in [
-                "blister pack",
-                "draft pack",
-                "draft set",
-                "multipack",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.DRAFT_SET
-
-        if any(
-            tag in product_name
-            for tag in [
-                "booster battle pack",
-                "clash pack",
-                "fourth edition gift box",
-                "quick start set",
-                "two player starter",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.TWO_PLAYER_STARTER_SET
-
-        if any(
-            tag in product_name
-            for tag in [
-                "box set",
-                "builders toolkit",
-                "commander collection",
-                "deckmasters tin",
-                "deluxe collection",
-                "edition box",
-                "game night",
-                "global series",
-                "hascon",
-                "modern event deck",
-                "planechase",
-                "planeswalker set",
-                "premium deck series",
-                "sdcc",
-                "secret lair",
-                "signature spellbook",
-            ]
-        ):
-            # In this section, only Planechase may have a "set of"
-            if "planechase" in product_name and "set of" in product_name:
-                return MtgjsonSealedProductCategory.CASE
-            return MtgjsonSealedProductCategory.BOX_SET
-
-        if "commander" in product_name or "brawl deck" in product_name:
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.CASE
-            return MtgjsonSealedProductCategory.COMMANDER_DECK
-
-        if "deck box" in product_name or "deck display" in product_name:
-            return MtgjsonSealedProductCategory.DECK_BOX
-
-        if any(
-            tag in product_name
-            for tag in [
-                "challenge deck",
-                "challenger deck",
-                "championship deck",
-                "event deck",
-                "guild kit",
-                "intro pack",
-                "planeswalker deck",
-                "starter deck",
-                "theme deck",
-                "tournament deck",
-                "tournament pack",
-            ]
-        ):
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.SUBSET
-            return MtgjsonSealedProductCategory.DECK
-
-        if any(
-            tag in product_name
-            for tag in [
-                "bundle",
-                "fat pack",
-                "gift box",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.BUNDLE
-
-        if "land station" in product_name:
-            return MtgjsonSealedProductCategory.LAND_STATION
-
-        return None
-
-    # Best-effort to parse the product name and determine the sealed product category
-    def determine_mtgjson_sealed_product_subtype(
-        self, product_name: str, category: Optional[MtgjsonSealedProductCategory]
-    ) -> Optional[MtgjsonSealedProductSubtype]:
-        """
-        Best-effort to parse the product name and determine the sealed product subtype
-        :param product_name Name of the product from TCG
-        :param category Category as parsed from determine_mtgjson_sealed_product_category()
-        :return: subtype
-        """
-        if not category or not category.value:
-            return None
-
-        for subtype in MtgjsonSealedProductSubtype:
-            if not subtype or not subtype.value:
-                continue
-
-            # Prevent aliasing from Eventide
-            if (
-                subtype is MtgjsonSealedProductSubtype.EVENT
-                and category is not MtgjsonSealedProductCategory.DECK
-            ):
-                continue
-
-            # Prevent assigning 'set' (for set boosters) to unrelated categories
-            if subtype is MtgjsonSealedProductSubtype.SET and (
-                category is not MtgjsonSealedProductCategory.BOOSTER_PACK
-                and category is not MtgjsonSealedProductCategory.BOOSTER_BOX
-            ):
-                continue
-
-            # Do the replace to use the tag as text
-            if subtype.value and subtype.value.replace("_", " ") in product_name:
-                return subtype
-
-        # Special handling because sometimes 'default' is not tagged
-        if category in [
-            MtgjsonSealedProductCategory.BOOSTER_BOX,
-            MtgjsonSealedProductCategory.BOOSTER_PACK,
-        ]:
-            return MtgjsonSealedProductSubtype.DEFAULT
-        return None
-
     def update_sealed_product(
         self, set_name: str, sealed_products: List[MtgjsonSealedProductObject]
     ) -> None:
@@ -340,6 +147,10 @@ class CardKingdomProvider(AbstractProvider):
             if product["edition"].lower() != updated_set_name:
                 continue
 
+            skip_products = ["pure bulk:", "complete set", "complete foil set"]
+            if any(s in product["name"].lower() for s in skip_products):
+                continue
+
             product_name = product["name"]
             for tag, fix in sealed_name_fixes.items():
                 if tag in product_name:
@@ -367,10 +178,10 @@ class CardKingdomProvider(AbstractProvider):
 
             sealed_product.identifiers.card_kingdom_id = str(product["id"])
 
-            sealed_product.category = self.determine_mtgjson_sealed_product_category(
+            sealed_product.category = sealed_product.determine_mtgjson_sealed_product_category(
                 sealed_product.name.lower()
             )
-            sealed_product.subtype = self.determine_mtgjson_sealed_product_subtype(
+            sealed_product.subtype = sealed_product.determine_mtgjson_sealed_product_subtype(
                 sealed_product.name.lower(), sealed_product.category
             )
 
diff --git a/mtgjson5/providers/gatherer.py b/mtgjson5/providers/gatherer.py
index 106a538..9904f47 100644
--- a/mtgjson5/providers/gatherer.py
+++ b/mtgjson5/providers/gatherer.py
@@ -72,6 +72,7 @@ class GathererProvider(AbstractProvider):
             urllib3.exceptions.MaxRetryError,
             requests.exceptions.RetryError,
             requests.exceptions.SSLError,
+            requests.exceptions.ConnectionError,
         ) as exception:
             LOGGER.warning(f"Unable to get {url} with {params}: {exception}")
             return None
diff --git a/mtgjson5/providers/github_decks.py b/mtgjson5/providers/github_decks.py
index fbf8ad2..db7f46b 100644
--- a/mtgjson5/providers/github_decks.py
+++ b/mtgjson5/providers/github_decks.py
@@ -5,10 +5,12 @@ import copy
 import json
 import logging
 import pathlib
+from collections import defaultdict
 from typing import Any, Dict, Iterator, List, Optional, Union
 
 from singleton_decorator import singleton
 
+from ..classes import MtgjsonSetDeckObject
 from ..classes.mtgjson_deck import MtgjsonDeckObject
 from ..compiled_classes.mtgjson_structures import MtgjsonStructuresObject
 from ..mtgjson_config import MtgjsonConfig
@@ -25,16 +27,19 @@ class GitHubDecksProvider(AbstractProvider):
     """
 
     decks_api_url: str = "https://github.com/taw/magic-preconstructed-decks-data/blob/master/decks_v2.json?raw=true"
+    decks_uuid_api_url: str = "https://github.com/mtgjson/mtg-sealed-content/blob/main/outputs/deck_map.json?raw=True"
     all_printings_file: pathlib.Path = MtgjsonConfig().output_path.joinpath(
         f"{MtgjsonStructuresObject().all_printings}.json"
     )
     all_printings_cards: Dict[str, Any]
+    decks_by_set: Dict[str, List[MtgjsonSetDeckObject]]
 
     def __init__(self) -> None:
         """
         Initializer
         """
         super().__init__(self._build_http_header())
+        self.decks_by_set = defaultdict(list)
 
     def _build_http_header(self) -> Dict[str, str]:
         """
@@ -43,6 +48,34 @@ class GitHubDecksProvider(AbstractProvider):
         """
         return {}
 
+    def get_decks_in_set(self, set_code: str) -> List[MtgjsonSetDeckObject]:
+        """
+        Get individual decks within a specific set, from cache
+        Builds up cache if not set
+        :param set_code Set code to get decks for
+        :return Decks in set code
+        """
+        if not self.decks_by_set:
+            decks_uuid_content = self.download(self.decks_uuid_api_url)
+            for deck in self.download(self.decks_api_url):
+                sealed_uuids = decks_uuid_content.get(set_code.lower(), {}).get(
+                    deck["name"]
+                )
+                mtgjson_set_deck = MtgjsonSetDeckObject(deck["name"], sealed_uuids)
+
+                for card in deck.get("cards", []):
+                    mtgjson_set_deck.cards.append(
+                        MtgjsonSetDeckObject.MtgjsonSetDeckCardObject(
+                            card["mtgjson_uuid"],
+                            card["count"],
+                            "foil" if card["foil"] else "nonfoil",
+                        )
+                    )
+
+                self.decks_by_set[deck.get("set_code").upper()].append(mtgjson_set_deck)
+
+        return self.decks_by_set.get(set_code, [])
+
     def download(
         self, url: str, params: Optional[Dict[str, Union[str, int]]] = None
     ) -> Any:
diff --git a/mtgjson5/providers/scryfall/set_language_detector.py b/mtgjson5/providers/scryfall/set_language_detector.py
index c5bc278..e2e500f 100644
--- a/mtgjson5/providers/scryfall/set_language_detector.py
+++ b/mtgjson5/providers/scryfall/set_language_detector.py
@@ -1,6 +1,8 @@
 import logging
+import time
 from typing import Any, Dict, List, Optional, Union
 
+import requests
 from singleton_decorator import singleton
 
 from ...constants import LANGUAGE_MAP
@@ -23,11 +25,25 @@ class ScryfallProviderSetLanguageDetector(AbstractProvider):
         return sf_utils.build_http_header()
 
     def download(
-        self, url: str, params: Optional[Dict[str, Union[str, int]]] = None
+        self,
+        url: str,
+        params: Optional[Dict[str, Union[str, int]]] = None,
+        retry_ttl: int = 3,
     ) -> Any:
         session = retryable_session()
-        response = session.get(url)
-        self.log_download(response)
+
+        try:
+            response = session.get(url)
+            self.log_download(response)
+        except requests.exceptions.ChunkedEncodingError as error:
+            if retry_ttl:
+                LOGGER.warning(f"Download failed: {error}... Retrying")
+                time.sleep(3 - retry_ttl)
+                return self.download(url, params, retry_ttl - 1)
+
+            LOGGER.error(f"Download failed: {error}... Maxed out retries")
+            return {}
+
         return response.json()
 
     def get_set_printing_languages(self, set_code: str) -> List[str]:
diff --git a/mtgjson5/providers/tcgplayer.py b/mtgjson5/providers/tcgplayer.py
index ded6a74..2677668 100644
--- a/mtgjson5/providers/tcgplayer.py
+++ b/mtgjson5/providers/tcgplayer.py
@@ -16,7 +16,6 @@ from ..classes import (
     MtgjsonPricesObject,
     MtgjsonSealedProductCategory,
     MtgjsonSealedProductObject,
-    MtgjsonSealedProductSubtype,
 )
 from ..mtgjson_config import MtgjsonConfig
 from ..providers.abstract import AbstractProvider
@@ -274,210 +273,6 @@ class TCGPlayerProvider(AbstractProvider):
 
         return dict(combined_listings)
 
-    def determine_mtgjson_sealed_product_category(
-        self, product_name: str
-    ) -> Optional[MtgjsonSealedProductCategory]:
-        """
-        Best-effort to parse the product name and determine the sealed product category
-        :param product_name Name of the product from TCG, must be lowercase
-        :return: category
-        """
-
-        # The order of the checks is important: for example, we need to catch all 'case'
-        # products before everything else, since "booster box case" would be caught in
-        # the 'booster box' check. The same applies to several sub products, such as
-        # 'prerelease' (vs guild kit), 'box set' (vs set boosters), and so on.
-        if any(
-            tag in product_name
-            for tag in [
-                "booster case",
-                "box case",
-                "bundle case",
-                "display case",
-                "intro display",
-                "intro pack display",
-                "pack box",
-                "pack case",
-                "tournament pack display",
-                "vip edition box",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.CASE
-
-        if any(
-            tag in product_name
-            for tag in ["booster box", "booster display", "mythic edition"]
-        ):
-            return MtgjsonSealedProductCategory.BOOSTER_BOX
-
-        # Needs to be before BOOSTER_PACK due to aliasing
-        if any(
-            tag in product_name
-            for tag in [
-                "blister pack",
-                "booster draft pack",
-                "booster packs draft set",
-                "multipack",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.DRAFT_SET
-
-        if any(
-            tag in product_name
-            for tag in [
-                "booster pack",
-                "booster retail pack",
-                "box topper",
-                "hanger pack",
-                "omega pack",
-                "promo pack",
-                "theme booster",
-                "vip edition pack",
-            ]
-        ):
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.SUBSET
-            return MtgjsonSealedProductCategory.BOOSTER_PACK
-
-        if "prerelease" in product_name:
-            return MtgjsonSealedProductCategory.PRERELEASE_PACK
-
-        if any(
-            tag in product_name
-            for tag in [
-                "booster battle pack",
-                "clash pack",
-                "fourth edition gift box",
-                "quick start set",
-                "two player starter",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.TWO_PLAYER_STARTER_SET
-
-        if any(
-            tag in product_name
-            for tag in [
-                "box set",
-                "builders toolkit",
-                "commander collection",
-                "deckmasters tin",
-                "deluxe collection",
-                "edition box",
-                "game night",
-                "global series",
-                "hascon",
-                "modern event deck",
-                "planechase",
-                "planeswalker set",
-                "premium deck series",
-                "sdcc",
-                "secret lair",
-                "signature spellbook",
-            ]
-        ):
-            # In this section, only Planechase may have a "set of"
-            if "planechase" in product_name and "set of" in product_name:
-                return MtgjsonSealedProductCategory.CASE
-            return MtgjsonSealedProductCategory.BOX_SET
-
-        if "commander" in product_name or "brawl deck" in product_name:
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.CASE
-            return MtgjsonSealedProductCategory.COMMANDER_DECK
-
-        if "deck box" in product_name or "deck display" in product_name:
-            return MtgjsonSealedProductCategory.DECK_BOX
-
-        if any(
-            tag in product_name
-            for tag in [
-                "challenge deck",
-                "challenger deck",
-                "championship deck",
-                "event deck",
-                "guild kit",
-                "intro pack",
-                "planeswalker deck",
-                "starter deck",
-                "theme deck",
-                "tournament deck",
-                "tournament pack",
-            ]
-        ):
-            if "set of" in product_name:
-                return MtgjsonSealedProductCategory.SUBSET
-            return MtgjsonSealedProductCategory.DECK
-
-        if any(
-            tag in product_name
-            for tag in [
-                "bundle",
-                "fat pack",
-                "gift box",
-            ]
-        ):
-            return MtgjsonSealedProductCategory.BUNDLE
-
-        if "land station" in product_name:
-            return MtgjsonSealedProductCategory.LAND_STATION
-
-        return None
-
-    # Best-effort to parse the product name and determine the sealed product category
-    def determine_mtgjson_sealed_product_subtype(
-        self, product_name: str, category: Optional[MtgjsonSealedProductCategory]
-    ) -> Optional[MtgjsonSealedProductSubtype]:
-        """
-        Best-effort to parse the product name and determine the sealed product subtype
-        :param product_name Name of the product from TCG
-        :param category Category as parsed from determine_mtgjson_sealed_product_category()
-        :return: subtype
-        """
-        if not category:
-            return None
-
-        for subtype in MtgjsonSealedProductSubtype:
-            if not subtype or not subtype.value:
-                continue
-
-            # Prevent aliasing from Eventide
-            if (
-                subtype is MtgjsonSealedProductSubtype.EVENT
-                and category is not MtgjsonSealedProductCategory.DECK
-            ):
-                continue
-
-            # Prevent assigning 'set' or 'draft_set' to Core Set editions
-            if "core set" in product_name and "set" in subtype.value:
-                continue
-
-            # Skip 'collector' for this set, since they weren't introduced it yet
-            if (
-                "ravnica allegiance" in product_name
-                and subtype is MtgjsonSealedProductSubtype.COLLECTOR
-            ):
-                continue
-
-            # Prevent assigning 'set' (for set boosters) to unrelated categories
-            if subtype is MtgjsonSealedProductSubtype.SET and (
-                category is not MtgjsonSealedProductCategory.BOOSTER_PACK
-                and category is not MtgjsonSealedProductCategory.BOOSTER_BOX
-            ):
-                continue
-
-            # Do the replace to use the tag as text
-            if subtype.value and subtype.value.replace("_", " ") in product_name:
-                return subtype
-
-        # Special handling because sometimes 'default' is not tagged
-        if category in [
-            MtgjsonSealedProductCategory.BOOSTER_BOX,
-            MtgjsonSealedProductCategory.BOOSTER_PACK,
-            MtgjsonSealedProductCategory.DRAFT_SET,
-        ]:
-            return MtgjsonSealedProductSubtype.DEFAULT
-        return None
-
     def generate_mtgjson_sealed_product_objects(
         self, group_id: Optional[int], set_code: str
     ) -> List[MtgjsonSealedProductObject]:
@@ -538,10 +333,10 @@ class TCGPlayerProvider(AbstractProvider):
             sealed_product.identifiers.tcgplayer_product_id = str(product["productId"])
             sealed_product.release_date = product["presaleInfo"].get("releasedOn")
 
-            sealed_product.category = self.determine_mtgjson_sealed_product_category(
+            sealed_product.category = sealed_product.determine_mtgjson_sealed_product_category(
                 sealed_product.name.lower()
             )
-            sealed_product.subtype = self.determine_mtgjson_sealed_product_subtype(
+            sealed_product.subtype = sealed_product.determine_mtgjson_sealed_product_subtype(
                 sealed_product.name.lower(), sealed_product.category
             )
 
@@ -637,7 +432,12 @@ def get_tcgplayer_sealed_data(group_id: Optional[int]) -> List[Dict[str, Any]]:
             # No more entries
             break
 
-        response = json.loads(api_response)
+        try:
+            response = json.loads(api_response)
+        except json.decoder.JSONDecodeError:
+            LOGGER.error(f"Unable to decode TCGPlayer API Response {api_response}")
+            break
+
         if not response["results"]:
             LOGGER.warning(f"Issue with Sealed Product for Group ID: {group_id}")
             break
diff --git a/mtgjson5/resources/cardkingdom_sealed_name_mapping.json b/mtgjson5/resources/cardkingdom_sealed_name_mapping.json
index 6ff8333..bdf502f 100644
--- a/mtgjson5/resources/cardkingdom_sealed_name_mapping.json
+++ b/mtgjson5/resources/cardkingdom_sealed_name_mapping.json
@@ -295,13 +295,20 @@
         "zendikar intro pack the adventurers": "zendikar intro pack green the adventurers",
         "odyssey theme deck trounceomatic": "odyssey theme deck trounce o matic",
         "odyssey theme deck onetwo punch": "odyssey theme deck one two punch",
-        "warhammer 40000 commander deck forces of the imperium": "universes beyond warhammer 40000 forces of the imperium commander deck",
-        "warhammer 40000 commander deck tyranid swarm": "universes beyond warhammer 40000 tyranid swarm commander deck",
-        "warhammer 40000 commander deck necron dynasties": "universes beyond warhammer 40000 necron dynasties commander deck",
-        "warhammer 40000 commander deck the ruinous powers": "universes beyond warhammer 40000 the ruinous powers commander deck",
-        "warhammer 40000 collectors edition commander deck tyranid swarm": "universes beyond warhammer 40000 tyranid swarm commander deck collectors edition",
-        "warhammer 40000 collectors edition commander deck the ruinous powers": "universes beyond warhammer 40000 the ruinous powers commander deck collectors edition",
-        "warhammer 40000 collectors edition commander deck necron dynasties": "universes beyond warhammer 40000 necron dynasties commander deck collectors edition",
-        "warhammer 40000 collectors edition commander deck forces of the imperium": "universes beyond warhammer 40000 forces of the imperium commander deck collectors edition"
+        "warhammer 40000 commander deck forces of the imperium": "warhammer 40000 forces of the imperium commander deck",
+        "warhammer 40000 commander deck tyranid swarm": "warhammer 40000 tyranid swarm commander deck",
+        "warhammer 40000 commander deck necron dynasties": "warhammer 40000 necron dynasties commander deck",
+        "warhammer 40000 commander deck the ruinous powers": "warhammer 40000 the ruinous powers commander deck",
+        "warhammer 40000 collectors edition commander deck tyranid swarm": "warhammer 40000 tyranid swarm commander deck collectors edition",
+        "warhammer 40000 collectors edition commander deck the ruinous powers": "warhammer 40000 the ruinous powers commander deck collectors edition",
+        "warhammer 40000 collectors edition commander deck necron dynasties": "warhammer 40000 necron dynasties commander deck collectors edition",
+        "warhammer 40000 collectors edition commander deck forces of the imperium": "warhammer 40000 forces of the imperium commander deck collectors edition",
+        "portal three kingdoms theme deck shu kingdom": "portal three kingdoms shu kingdom theme deck",
+        "portal three kingdoms theme deck wei kingdom": "portal three kingdoms wei kingdom theme deck",
+        "portal three kingdoms theme deck wu kingdom": "portal three kingdoms wu kingdom theme deck",
+        "commander 2018 adaptive enchantment": "commander 2018 deck adaptive enchantment",
+        "commander 2018 exquisite invention": "commander 2018 deck exquisite invention",
+        "commander 2018 natures vengeance": "commander 2018 deck natures vengeance",
+        "commander 2018 subjective reality": "commander 2018 deck subjective reality"
     }
-}
\ No newline at end of file
+}
diff --git a/mtgjson5/resources/sealed_name_fixes.json b/mtgjson5/resources/sealed_name_fixes.json
index c34de31..14f3688 100644
--- a/mtgjson5/resources/sealed_name_fixes.json
+++ b/mtgjson5/resources/sealed_name_fixes.json
@@ -35,12 +35,19 @@
     "Magic 2013": "2013 Core Set",
     "Magic 2014": "2014 Core Set",
     "Magic 2015": "2015 Core Set",
+    "Universes Beyond ": "",
+    "Portal: Three Kingdoms": "Portal Three Kingdoms",
     "The Brothers War": "The Brothers' War",
     "Shards of Alara Block": "Shards of Alara",
+    "Secret Lair Drop": "Secret Lair",
+    "Secret Lair:": "Secret Lair",
+    "The Lord of the Rings: Tales of Middle-earth": "The Lord of the Rings Tales of Middle earth",
 
     "COMMENT3": "Resolve 'deck' vs 'decks'",
     "Decks - Set": "\u2691",
+    "Deck - Set": "\u2691",
     "Decks Set": "\u2691",
+    "Deck Set": "\u2691",
     "Decks": "Deck",
     "\u2691": "Decks - Set",
 
diff --git a/mtgjson5/set_builder.py b/mtgjson5/set_builder.py
index b59e298..aab4a8d 100644
--- a/mtgjson5/set_builder.py
+++ b/mtgjson5/set_builder.py
@@ -30,6 +30,7 @@ from .providers import (
     FandomProviderSecretLair,
     GathererProvider,
     GitHubBoostersProvider,
+    GitHubDecksProvider,
     GitHubSealedProvider,
     MTGBanProvider,
     MultiverseBridgeProvider,
@@ -495,6 +496,10 @@ def build_mtgjson_set(set_code: str) -> Optional[MtgjsonSetObject]:
 
     mark_duel_decks(set_code, mtgjson_set.cards)
 
+    mtgjson_set.decks = GitHubDecksProvider().get_decks_in_set(set_code)
+    for mtgjson_set_deck in mtgjson_set.decks:
+        mtgjson_set_deck.add_sealed_product_uuids(mtgjson_set.sealed_product)
+
     if "Art Series" in mtgjson_set.name:
         add_orientations(mtgjson_set)
 


Test patch:
diff --git a/tests/mtgjson5/providers/test_tcgplayer.py b/tests/mtgjson5/providers/test_tcgplayer.py
index 214d8c7..e32a212 100644
--- a/tests/mtgjson5/providers/test_tcgplayer.py
+++ b/tests/mtgjson5/providers/test_tcgplayer.py
@@ -3,8 +3,7 @@ from typing import Tuple, List
 
 import pytest
 
-from mtgjson5.classes import MtgjsonSealedProductCategory
-from mtgjson5.providers import TCGPlayerProvider
+from mtgjson5.classes import MtgjsonSealedProductCategory, MtgjsonSealedProductObject
 
 testdata: List[Tuple[str, MtgjsonSealedProductCategory]] = [
     ("Double Masters - Booster Box Case", MtgjsonSealedProductCategory.CASE),
@@ -36,4 +35,4 @@ testdata: List[Tuple[str, MtgjsonSealedProductCategory]] = [
 @pytest.mark.parametrize("product_name,expected", testdata)
 def test_determine_mtgjson_sealed_product_category(product_name, expected):
     """Test the function that decides what the type is for a sealed product"""
-    assert TCGPlayerProvider().determine_mtgjson_sealed_product_category(product_name.lower()) == expected
+    assert MtgjsonSealedProductObject().determine_mtgjson_sealed_product_category(product_name.lower()) == expected