Repository: cancervariants/disease-normalization (language: python)

Issue:
build: use latest version of ga4gh.vrsatile.pydantic
close #82


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml
index c325d687..9ffe6fdb 100644
--- a/.github/workflows/github-actions.yml
+++ b/.github/workflows/github-actions.yml
@@ -1,37 +1,59 @@
 name: github-actions
 on: [push, pull_request]
 jobs:
-    test:
-        runs-on: ubuntu-latest
-        env:
-            AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
-            AWS_DEFAULT_REGION: us-east-2
-            AWS_DEFAULT_OUTPUT: text
-            DISEASE_NORM_DB_URL: http://localhost:8000
-            TEST: DISEASE_TEST
-        steps:
-            - uses: actions/checkout@v2
+  test:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      AWS_DEFAULT_OUTPUT: text
+      DISEASE_NORM_DB_URL: http://localhost:8000
+      TEST: DISEASE_TEST
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+    steps:
+    - uses: actions/checkout@v3
 
-            - name: Setup Python
-              uses: actions/setup-python@v1
-              with:
-                  python-version: 3.8
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+          python-version: ${{ matrix.python-version }}
 
-            - name: Install dependencies
-              run: |
-                  python3 -m pip install pipenv
-                  pipenv install --dev
+    - name: Install dependencies
+      run: |
+          python3 -m pip install pipenv
+          pipenv install --dev
 
-            - name: Build local DynamoDB
-              run: |
-                  chmod +x ./tests/unit/dynamodb_build.bash
-                  ./tests/unit/dynamodb_build.bash
+    - name: Build local DynamoDB
+      run: |
+          chmod +x ./tests/unit/dynamodb_build.bash
+          ./tests/unit/dynamodb_build.bash
 
-            - name: Load and Test DynamoDB
-              run: |
-                  pipenv run pytest tests/unit/test_database.py
+    - name: Load and Test DynamoDB
+      run: |
+          pipenv run pytest tests/unit/test_database.py
 
-            - run: pipenv run flake8
-            - run: pipenv run pytest tests/
+    - run: pipenv run pytest tests/
+  lint:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      AWS_DEFAULT_OUTPUT: text
+    steps:
+    - uses: actions/checkout@v3
 
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+          python3 -m pip install pipenv
+          pipenv install --dev
+    - name: check style
+      run: pipenv run flake8
diff --git a/Pipfile b/Pipfile
index 4125c5a1..9458a0e6 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,7 +5,8 @@ name = "pypi"
 
 [dev-packages]
 disease-normalizer = {editable = true, path = "."}
-click = "*"
+bioversions = "*"
+requests = "*"
 owlready2 = "*"
 rdflib = "*"
 pytest = "*"
@@ -24,3 +25,4 @@ fastapi = ">=0.72.0"
 uvicorn = "*"
 boto3 = "*"
 "ga4gh.vrsatile.pydantic" = "==0.0.11"
+click = "*"
diff --git a/disease/__init__.py b/disease/__init__.py
index 711c9a88..e8bb62e8 100644
--- a/disease/__init__.py
+++ b/disease/__init__.py
@@ -5,7 +5,7 @@
 from .version import __version__  # noqa: F401
 
 
-PROJECT_ROOT = Path(__file__).resolve().parents[0]
+APP_ROOT = Path(__file__).resolve().parents[0]
 
 # set up logging
 logging.basicConfig(
diff --git a/disease/cli.py b/disease/cli.py
index e1ba8a5d..d6048160 100644
--- a/disease/cli.py
+++ b/disease/cli.py
@@ -1,21 +1,18 @@
 """This module provides a CLI util to make updates to normalizer database."""
 from timeit import default_timer as timer
 from os import environ
+from typing import Optional, List
 
 import click
 from botocore.exceptions import ClientError
 from boto3.dynamodb.conditions import Key
 
-from disease import SOURCES_LOWER_LOOKUP, logger
+from disease import logger
 from disease.schemas import SourceName
+from disease.etl.merge import Merge
 from disease.database import Database, confirm_aws_db_use, SKIP_AWS_DB_ENV_NAME, \
     VALID_AWS_ENV_NAMES, AWS_ENV_VAR_NAME
-from disease.etl.merge import Merge
-from disease.etl.mondo import Mondo
-from disease.etl import NCIt  # noqa: F401
-from disease.etl import DO  # noqa: F401
-from disease.etl import OncoTree  # noqa: F401
-from disease.etl import OMIM  # noqa: F401
+from disease.etl import Mondo, NCIt, DO, OncoTree, OMIM  # noqa: F401
 
 
 # Use to lookup class object from source name. Should be one key-value pair
@@ -50,11 +47,16 @@ class CLI:
     @click.option(
         '--update_merged',
         is_flag=True,
-        help='Update concepts for normalize endpoint. Must select either '
-             '--update_all or include Mondo as a normalizer source argument.'
+        help='Update concepts for /normalize endpoint.'
+    )
+    @click.option(
+        '--from_local',
+        is_flag=True,
+        default=False,
+        help="Use most recent local source data instead of fetching latest versions."
     )
     def update_normalizer_db(normalizer, aws_instance, db_url, update_all,
-                             update_merged):
+                             update_merged, from_local):
         """Update selected source(s) in the Disease Normalizer database."""
         # If SKIP_AWS_CONFIRMATION is accidentally set, we should verify that the
         # aws instance should actually be used
@@ -69,86 +71,128 @@ def update_normalizer_db(normalizer, aws_instance, db_url, update_all,
         else:
             if db_url:
                 endpoint_url = db_url
-            elif 'DISEASE_NORM_DB_URL' in environ.keys():
-                endpoint_url = environ['DISEASE_NORM_DB_URL']
+            elif "DISEASE_NORM_DB_URL" in environ:
+                endpoint_url = environ["DISEASE_NORM_DB_URL"]
             else:
-                endpoint_url = 'http://localhost:8000'
-            db: Database = Database(db_url=endpoint_url)
+                endpoint_url = "http://localhost:8000"
+            db = Database(db_url=endpoint_url)
 
         if update_all:
-            normalizers = list(src for src in SOURCES_CLASS_LOOKUP)
-            CLI()._update_normalizers(normalizers, db, update_merged)
+            sources_to_update = list(src for src in SOURCES_CLASS_LOOKUP)
+            CLI()._update_sources(sources_to_update, db, update_merged, from_local)
         elif not normalizer:
-            CLI()._help_msg("Must provide 1 or more source names, or use `--update_all` parameter")  # noqa: E501
+            if update_merged:
+                CLI()._update_merged(db, [])
+            else:
+                CLI()._help_msg()
         else:
-            normalizers = normalizer.lower().split()
-
-            if len(normalizers) == 0:
-                CLI()._help_msg("Must provide 1 or more source names, or use `--update_all` parameter")  # noqa: E501
-
-            non_sources = set(normalizers) - {src for src
-                                              in SOURCES_LOWER_LOOKUP}
-
-            if len(non_sources) != 0:
-                raise Exception(f"Not valid source(s): {non_sources}. \n"
-                                f"Legal sources are "
-                                f"{list(SOURCES_LOWER_LOOKUP.values())}.")
+            sources_to_update = str(normalizer).lower().split()
+            if len(sources_to_update) == 0:
+                CLI()._help_msg()
 
-            if update_merged and 'mondo' not in normalizers:
-                CLI()._help_msg("Must include Mondo in sources to update for `--update_merged`")  # noqa: E501
+            invalid_sources = set(sources_to_update) - {src for src
+                                                        in SOURCES_CLASS_LOOKUP}
+            if len(invalid_sources) != 0:
+                raise Exception(f"Not valid sources: {invalid_sources}")
 
-            CLI()._update_normalizers(normalizers, db, update_merged)
+            CLI()._update_sources(sources_to_update, db, update_merged, from_local)
 
     @staticmethod
-    def _help_msg(message):
+    def _help_msg(message: Optional[str] = None):
         """Display help message."""
         ctx = click.get_current_context()
-        click.echo(message)
+        if message:
+            click.echo(message)
         click.echo(ctx.get_help())
         ctx.exit()
 
     @staticmethod
-    def _update_normalizers(normalizers, db, update_merged):
+    def _update_sources(sources: List[str], db: Database, update_merged: bool,
+                        from_local: bool = False):
         """Update selected normalizer sources."""
-        processed_ids = []
-        for n in normalizers:
-            msg = f"Deleting {n}..."
+        added_ids = []
+        for source in sources:
+            msg = f"Deleting {source}..."
             click.echo(f"\n{msg}")
             logger.info(msg)
             start_delete = timer()
-            CLI()._delete_data(n, db)
+            CLI()._delete_data(source, db)
             end_delete = timer()
             delete_time = end_delete - start_delete
-            msg = f"Deleted {n} in {delete_time:.5f} seconds."
+            msg = f"Deleted {source} in {delete_time:.5f} seconds."
             click.echo(f"{msg}\n")
             logger.info(msg)
 
-            msg = f"Loading {n}..."
+            msg = f"Loading {source}..."
             click.echo(msg)
             logger.info(msg)
             start_load = timer()
-            source = SOURCES_CLASS_LOOKUP[n](database=db)
+            source = SOURCES_CLASS_LOOKUP[source](database=db)
             if isinstance(source, Mondo):
-                processed_ids = source.perform_etl()
+                added_ids = source.perform_etl(from_local)
             else:
                 source.perform_etl()
             end_load = timer()
             load_time = end_load - start_load
-            msg = f"Loaded {n} in {load_time:.5f} seconds."
+            msg = f"Loaded {source} in {load_time:.5f} seconds."
             click.echo(msg)
             logger.info(msg)
-            msg = f"Total time for {n}: " \
+            msg = f"Total time for {source}: " \
                   f"{(delete_time + load_time):.5f} seconds."
             click.echo(msg)
             logger.info(msg)
-        if update_merged and processed_ids:
-            click.echo("Generating merged concepts...")
-            merge = Merge(database=db)
-            merge.create_merged_concepts(processed_ids)
-            click.echo("Merged concept generation complete.")
+        if update_merged:
+            CLI()._update_merged(db, added_ids)
+
+    def _update_merged(self, db: Database, added_ids: List[str]) -> None:
+        """Update merged concepts and references.
+        :param db: Database client
+        :param added_ids: list of concept IDs to use for normalized groups.
+            Should consist solely of MONDO IDs. If empty, will be fetched.
+        """
+        start_merge = timer()
+        if not added_ids:
+            CLI()._delete_merged_data(db)
+            added_ids = db.get_ids_for_merge()
+        merge = Merge(database=db)
+        click.echo("Constructing normalized records...")
+        merge.create_merged_concepts(added_ids)
+        end_merge = timer()
+        click.echo(f"Merged concept generation completed in"
+                   f" {(end_merge - start_merge):.5f} seconds.")
+
+    @staticmethod
+    def _delete_merged_data(database: Database):
+        """Delete data pertaining to merged records.
+        :param database: DynamoDB client
+        """
+        click.echo("\nDeleting normalized records...")
+        start_delete = timer()
+        try:
+            while True:
+                with database.diseases.batch_writer(
+                        overwrite_by_pkeys=["label_and_type", "concept_id"]) \
+                        as batch:
+                    response = database.diseases.query(
+                        IndexName="type_index",
+                        KeyConditionExpression=Key("item_type").eq("merger"),
+                    )
+                    records = response["Items"]
+                    if not records:
+                        break
+                    for record in records:
+                        batch.delete_item(Key={
+                            "label_and_type": record["label_and_type"],
+                            "concept_id": record["concept_id"]
+                        })
+        except ClientError as e:
+            click.echo(e.response["Error"]["Message"])
+        end_delete = timer()
+        delete_time = end_delete - start_delete
+        click.echo(f"Deleted normalized records in {delete_time:.5f} seconds.")
 
     @staticmethod
-    def _delete_data(source, database):
+    def _delete_data(source: str, database: Database):
         # Delete source's metadata
         try:
             metadata = database.metadata.query(
diff --git a/disease/database.py b/disease/database.py
index ce9eec17..7a48dc5c 100644
--- a/disease/database.py
+++ b/disease/database.py
@@ -259,6 +259,32 @@ def get_records_by_type(self, query: str,
                          f"{e.response['Error']['Message']}")
             return []
 
+    def get_ids_for_merge(self) -> List[str]:
+        """Retrieve concept IDs for use in generating normalized records.
+        :return: List of concept IDs as strings.
+        """
+        last_evaluated_key = None
+        concept_ids = []
+        params = {
+            "ProjectionExpression": "concept_id,item_type,src_name",
+        }
+        while True:
+            if last_evaluated_key:
+                response = self.diseases.scan(
+                    ExclusiveStartKey=last_evaluated_key, **params
+                )
+            else:
+                response = self.diseases.scan(**params)
+            records = response["Items"]
+            for record in records:
+                if record["item_type"] == "identity" and record["src_name"] == "Mondo":
+                    concept_id = record["concept_id"]
+                    concept_ids.append(concept_id)
+            last_evaluated_key = response.get("LastEvaluatedKey")
+            if not last_evaluated_key:
+                break
+        return concept_ids
+
     def add_record(self, record: Dict, record_type: str = "identity"):
         """Add new record to database.
 
diff --git a/disease/etl/base.py b/disease/etl/base.py
index e3177eb9..858283c2 100644
--- a/disease/etl/base.py
+++ b/disease/etl/base.py
@@ -1,47 +1,181 @@
 """A base class for extraction, transformation, and loading of data."""
+import tempfile
+import zipfile
+import os
 from abc import ABC, abstractmethod
-from disease import SOURCES_FOR_MERGE, ITEM_TYPES, logger
+from typing import Set, Dict, List, Optional, Callable
+from pathlib import Path
+import re
+import ftplib
+
+import owlready2 as owl
+import bioversions
+import requests
+
+from disease import SOURCES_FOR_MERGE, ITEM_TYPES, logger, APP_ROOT
 from disease.database import Database
 from disease.schemas import Disease
-import owlready2 as owl
-from typing import Set, Dict, List
-from pathlib import Path
+from disease.etl.utils import DownloadException
+
+
+DEFAULT_DATA_PATH = APP_ROOT / "data"
 
 
 class Base(ABC):
     """The ETL base class."""
 
-    def __init__(self, database: Database, data_path: Path):
-        """Extract from sources."""
+    def __init__(self, database: Database, data_path: Path = DEFAULT_DATA_PATH):
+        """Extract from sources.
+        :param database: DDB client
+        :param data_path: location of data directory
+        """
+        self.name = self.__class__.__name__.lower()
         self.database = database
-        self._data_path = data_path
+        self._src_dir: Path = Path(data_path / self.name)
         self._store_ids = self.__class__.__name__ in SOURCES_FOR_MERGE
         if self._store_ids:
-            self._processed_ids = []
+            self._added_ids = []
 
-    @abstractmethod
-    def perform_etl(self) -> List:
+    def perform_etl(self, use_existing: bool = False) -> List:
         """Public-facing method to begin ETL procedures on given data.
-
+        :param use_existing: if True, use local data instead of retrieving most recent
+        version
         :return: List of concept IDs to be added to merge generation.
         """
-        raise NotImplementedError
+        self._extract_data(use_existing)
+        self._load_meta()
+        self._transform_data()
+        if self._store_ids:
+            return self._added_ids
+        else:
+            return []
+
+    def get_latest_version(self) -> str:
+        """Get most recent version of source data. Should be overriden by
+        sources not added to Bioversions yet, or other special-case sources.
+        :return: most recent version, as a str
+        """
+        return bioversions.get_version(self.name)
 
+    @abstractmethod
     def _download_data(self):
         """Download source data."""
         raise NotImplementedError
 
-    def _extract_data(self):
-        """Get source file from data directory."""
-        self._data_path.mkdir(exist_ok=True, parents=True)
-        src_name = f'{type(self).__name__.lower()}_'
-        dir_files = [f for f in self._data_path.iterdir()
-                     if f.name.startswith(src_name)]
-        if len(dir_files) == 0:
+    def _zip_handler(self, dl_path: Path, outfile_path: Path) -> None:
+        """Provide simple callback function to extract the largest file within a given
+        zipfile and save it within the appropriate data directory.
+        :param Path dl_path: path to temp data file
+        :param Path outfile_path: path to save file within
+        """
+        with zipfile.ZipFile(dl_path, "r") as zip_ref:
+            if len(zip_ref.filelist) > 1:
+                files = sorted(zip_ref.filelist, key=lambda z: z.file_size,
+                               reverse=True)
+                target = files[0]
+            else:
+                target = zip_ref.filelist[0]
+            target.filename = outfile_path.name
+            zip_ref.extract(target, path=outfile_path.parent)
+        os.remove(dl_path)
+
+    @staticmethod
+    def _http_download(url: str, outfile_path: Path, headers: Optional[Dict] = None,
+                       handler: Optional[Callable[[Path, Path], None]] = None) -> None:
+        """Perform HTTP download of remote data file.
+        :param str url: URL to retrieve file from
+        :param Path outfile_path: path to where file should be saved. Must be an actual
+            Path instance rather than merely a pathlike string.
+        :param Optional[Dict] headers: Any needed HTTP headers to include in request
+        :param Optional[Callable[[Path, Path], None]] handler: provide if downloaded
+            file requires additional action, e.g. it's a zip file.
+        """
+        if handler:
+            dl_path = Path(tempfile.gettempdir()) / "therapy_dl_tmp"
+        else:
+            dl_path = outfile_path
+        # use stream to avoid saving download completely to memory
+        with requests.get(url, stream=True, headers=headers) as r:
+            try:
+                r.raise_for_status()
+            except requests.HTTPError:
+                raise DownloadException(
+                    f"Failed to download {outfile_path.name} from {url}."
+                )
+            with open(dl_path, "wb") as h:
+                for chunk in r.iter_content(chunk_size=8192):
+                    if chunk:
+                        h.write(chunk)
+        if handler:
+            handler(dl_path, outfile_path)
+
+    def _ftp_download(self, host: str, host_dir: str, host_fn: str) -> None:
+        """Download data file from FTP site.
+        :param str host: Source's FTP host name
+        :param str host_dir: Data directory located on FTP site
+        :param str host_fn: Filename on FTP site to be downloaded
+        """
+        try:
+            with ftplib.FTP(host) as ftp:
+                ftp.login()
+                logger.debug(f"FTP login to {host} was successful")
+                ftp.cwd(host_dir)
+                with open(self._src_dir / host_fn, "wb") as fp:
+                    ftp.retrbinary(f"RETR {host_fn}", fp.write)
+        except ftplib.all_errors as e:
+            logger.error(f"FTP download failed: {e}")
+            raise Exception(e)
+
+    def _parse_version(self, file_path: Path, pattern: Optional[re.Pattern] = None
+                       ) -> str:
+        """Get version number from provided file path.
+        :param Path file_path: path to located source data file
+        :param Optional[re.Pattern] pattern: regex pattern to use
+        :return: source data version
+        :raises: FileNotFoundError if version parsing fails
+        """
+        if pattern is None:
+            pattern = re.compile(type(self).__name__.lower() + r"_(.+)\..+")
+        matches = re.match(pattern, file_path.name)
+        if matches is None:
+            raise FileNotFoundError
+        else:
+            return matches.groups()[0]
+
+    def _get_latest_data_file(self) -> Path:
+        """Acquire most recent source data file."""
+        self._version = self.get_latest_version()
+        fglob = f"{self.name}_{self._version}.*"
+        latest = list(self._src_dir.glob(fglob))
+        if not latest:
             self._download_data()
-            dir_files = list(self._data_path.iterdir())
-        self._data_file = sorted(dir_files, reverse=True)[0]
-        self._version = self._data_file.stem.split('_', 1)[1]
+            latest = list(self._src_dir.glob(fglob))
+        assert len(latest) != 0  # probably unnecessary, but just to be safe
+        return latest[0]
+
+    def _extract_data(self, use_existing: bool = False) -> None:
+        """Get source file from data directory.
+        :param use_existing: if True, use local data regardless of whether it's up to
+            date
+        """
+        self._src_dir.mkdir(exist_ok=True, parents=True)
+        src_name = type(self).__name__.lower()
+        if use_existing:
+            files = list(sorted(self._src_dir.glob(f"{src_name}_*.*")))
+            if len(files) < 1:
+                raise FileNotFoundError(f"No source data found for {src_name}")
+            self._data_file: Path = files[-1]
+            try:
+                self._version = self._parse_version(self._data_file)
+            except FileNotFoundError:
+                raise FileNotFoundError(
+                    f"Unable to parse version value from {src_name} source data file "
+                    f"located at {self._data_file.absolute().as_uri()} -- "
+                    "check filename against schema defined in README: "
+                    "https://github.com/cancervariants/therapy-normalization#update-sources"  # noqa: E501
+                )
+        else:
+            self._data_file = self._get_latest_data_file()
 
     @abstractmethod
     def _transform_data(self, *args, **kwargs):
@@ -83,7 +217,7 @@ def _load_disease(self, disease: Dict):
 
         self.database.add_record(disease)
         if self._store_ids:
-            self._processed_ids.append(concept_id)
+            self._added_ids.append(concept_id)
 
 
 class OWLBase(Base):
diff --git a/disease/etl/do.py b/disease/etl/do.py
index c18d715c..57100461 100644
--- a/disease/etl/do.py
+++ b/disease/etl/do.py
@@ -1,13 +1,10 @@
 """Disease Ontology ETL module."""
-from .base import OWLBase
-import requests
-from pathlib import Path
-from disease import PROJECT_ROOT, PREFIX_LOOKUP, logger
-from disease.schemas import SourceMeta, SourceName, NamespacePrefix
-from disease.database import Database
-from datetime import datetime
 import owlready2 as owl
-from typing import List
+import bioversions
+
+from disease import PREFIX_LOOKUP, logger
+from disease.schemas import SourceMeta, SourceName, NamespacePrefix
+from disease.etl.base import OWLBase
 
 
 DO_PREFIX_LOOKUP = {
@@ -29,48 +26,20 @@
 class DO(OWLBase):
     """Disease Ontology ETL class."""
 
-    def __init__(self,
-                 database: Database,
-                 src_dload_page: str = "http://www.obofoundry.org/ontology/doid.html",  # noqa: E501
-                 src_url: str = "http://purl.obolibrary.org/obo/doid/doid-merged.owl",  # noqa: E501
-                 data_path: Path = PROJECT_ROOT / 'data' / 'do'):
-        """Override base class init method.
-
-        :param therapy.database.Database database: app database instance
-        :param str src_dload_page: URL for source data download webpage
-        :param str src_url: URL for source data file
-        :param pathlib.Path data_path: path to local DO data directory
-        """
-        self._SRC_DLOAD_PAGE = src_dload_page
-        self._SRC_URL = src_url
-        self._version = datetime.strftime(datetime.now(), "%Y%m%d")
-        super().__init__(database=database, data_path=data_path)
-
-    def perform_etl(self) -> List[str]:
-        """Public-facing method to initiate ETL procedures on given data.
-
-        :return: empty list (because DO IDs shouldn't be used to construct
-            merged concept groups)
+    def get_latest_version(self) -> str:
+        """Get most recent version of source data. Should be overriden by
+        sources not added to Bioversions yet, or other special-case sources.
+        :return: most recent version, as a str
         """
-        self._extract_data()
-        self._load_meta()
-        self._transform_data()
-        self.database.flush_batch()
-        return []
+        return bioversions.get_version("disease ontology")
 
     def _download_data(self):
         """Download DO source file for loading into normalizer."""
-        logger.info('Downloading DO data...')
-        try:
-            response = requests.get(self._SRC_URL, stream=True)
-        except requests.exceptions.RequestException as e:
-            logger.error(f'DO download failed: {e}')
-            raise e
-        handle = open(self._data_path / f'do_{self._version}.owl', "wb")
-        for chunk in response.iter_content(chunk_size=512):
-            if chunk:
-                handle.write(chunk)
-        logger.info('Finished downloading Human Disease Ontology')
+        logger.info('Retrieving source data for Disease Ontology')
+        output_file = self._src_dir / f"do_{self._version}.owl"
+        self._http_download("http://purl.obolibrary.org/obo/doid/doid-merged.owl",
+                            output_file)
+        logger.info("Successfully retrieved source data for Disease Ontology")
 
     def _load_meta(self):
         """Load metadata"""
@@ -78,7 +47,7 @@ def _load_meta(self):
             "data_license": "CC0 1.0",
             "data_license_url": "https://creativecommons.org/publicdomain/zero/1.0/legalcode",  # noqa: E501
             "version": self._version,
-            "data_url": self._SRC_DLOAD_PAGE,
+            "data_url": "http://www.obofoundry.org/ontology/doid.html",
             "rdp_url": None,
             "data_license_attributes": {
                 "non_commercial": False,
diff --git a/disease/etl/mondo.py b/disease/etl/mondo.py
index 9d034e5c..c78ea29d 100644
--- a/disease/etl/mondo.py
+++ b/disease/etl/mondo.py
@@ -1,12 +1,9 @@
 """Module to load disease data from Mondo Disease Ontology."""
+import owlready2 as owl
+
 from .base import OWLBase
-from disease import PROJECT_ROOT, PREFIX_LOOKUP, logger
-from disease.database import Database
+from disease import PREFIX_LOOKUP, logger
 from disease.schemas import SourceMeta, SourceName, NamespacePrefix
-from pathlib import Path
-import requests
-import owlready2 as owl
-from typing import List
 
 
 MONDO_PREFIX_LOOKUP = {
@@ -49,47 +46,12 @@
 class Mondo(OWLBase):
     """Gather and load data from Mondo."""
 
-    def __init__(self,
-                 database: Database,
-                 src_dload_page: str = "https://mondo.monarchinitiative.org/pages/download/",  # noqa: E501
-                 src_url: str = "http://purl.obolibrary.org/obo/mondo.owl",
-                 version: str = "20210129",
-                 data_path: Path = PROJECT_ROOT / 'data' / 'mondo'):
-        """Override base class init method.
-
-        :param therapy.database.Database database: app database instance
-        :param str src_dload_page: user-facing download page
-        :param str src_url: direct URL to OWL file download
-        :param pathlib.Path data_path: path to local Mondo data directory
-        """
-        self._SRC_DLOAD_PAGE = src_dload_page
-        self._SRC_URL = src_url
-        self._version = version
-        super().__init__(database=database, data_path=data_path)
-
-    def perform_etl(self) -> List[str]:
-        """Public-facing method to initiate ETL procedures on given data.
-
-        :return: List of concept IDs that were added.
-        """
-        self._extract_data()
-        self._load_meta()
-        self._transform_data()
-        self.database.flush_batch()
-        return self._processed_ids
-
     def _download_data(self):
         """Download Mondo thesaurus source file for loading into normalizer."""
         logger.info('Downloading Mondo data...')
-        try:
-            response = requests.get(self._SRC_URL, stream=True)
-        except requests.exceptions.RequestException as e:
-            logger.error(f'Mondo download failed: {e}')
-            raise e
-        handle = open(self._data_path / f'mondo_{self._version}.owl', "wb")
-        for chunk in response.iter_content(chunk_size=512):
-            if chunk:
-                handle.write(chunk)
+        url = "http://purl.obolibrary.org/obo/mondo.owl"
+        output_file = self._src_dir / f"mondo_{self._version}.owl"
+        self._http_download(url, output_file)
         logger.info('Finished downloading Mondo Disease Ontology')
 
     def _load_meta(self):
@@ -97,7 +59,7 @@ def _load_meta(self):
         metadata = SourceMeta(data_license="CC BY 4.0",
                               data_license_url="https://creativecommons.org/licenses/by/4.0/legalcode",  # noqa: E501
                               version=self._version,
-                              data_url=self._SRC_DLOAD_PAGE,
+                              data_url="https://mondo.monarchinitiative.org/pages/download/",  # noqa: E501
                               rdp_url='http://reusabledata.org/monarch.html',
                               data_license_attributes={
                                   'non_commercial': False,
diff --git a/disease/etl/ncit.py b/disease/etl/ncit.py
index 45f45842..81251471 100644
--- a/disease/etl/ncit.py
+++ b/disease/etl/ncit.py
@@ -1,16 +1,14 @@
 """Module to load disease data from NCIt."""
-from .base import OWLBase
-from disease import PROJECT_ROOT, logger
-from disease.database import Database
-from disease.schemas import SourceMeta, SourceName, NamespacePrefix
-from pathlib import Path
 import requests
-import zipfile
-from os import remove, rename
-from typing import Set, List
+from typing import Set
 import owlready2 as owl
 import re
 
+from disease import logger
+from disease.schemas import SourceMeta, SourceName, NamespacePrefix
+from disease.etl.base import OWLBase
+from disease.etl.utils import DownloadException
+
 
 icdo_re = re.compile("[0-9]+/[0-9]+")
 
@@ -18,63 +16,48 @@
 class NCIt(OWLBase):
     """Gather and load data from NCIt."""
 
-    def __init__(self,
-                 database: Database,
-                 src_dir: str = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/21.01d_Release/",  # noqa F401
-                 src_fname: str = "Thesaurus_21.01d.OWL.zip",
-                 data_path: Path = PROJECT_ROOT / 'data' / 'ncit'):
-        """Override base class init method.
-
-        :param therapy.database.Database database: app database instance
-        :param str src_dir: URL of remote directory containing source input
-        :param str src_fname: filename for source file within source directory
-        :param pathlib.Path data_path: path to local NCIt data directory
+    def _download_data(self) -> None:
+        """Download NCI thesaurus source file.
+        The NCI directory structure can be a little tricky, so this method attempts to
+        retrieve a file matching the latest version number from both the subdirectory
+        root (where the current version is typically posted) as well as the year-by-year
+        archives if that fails.
         """
-        self._SRC_DIR = src_dir
-        self._SRC_FNAME = src_fname
-        super().__init__(database=database, data_path=data_path)
-
-    def perform_etl(self) -> List[str]:
-        """Public-facing method to initiate ETL procedures on given data.
-
-        :return: empty list (because NCIt IDs shouldn't be used to construct
-            merged concept groups)
-        """
-        self._extract_data()
-        self._load_meta()
-        self._transform_data()
-        self.database.flush_batch()
-        return []
+        logger.info("Retrieving source data for NCIt")
+        base_url = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus"
+        # ping base NCIt directory
+        release_fname = f"Thesaurus_{self._version}.OWL.zip"
+        src_url = f"{base_url}/{release_fname}"
+        r_try = requests.get(src_url)
+        if r_try.status_code != 200:
+            # ping NCIt archive directories
+            archive_url = f"{base_url}/archive/{self._version}_Release/{release_fname}"
+            archive_try = requests.get(archive_url)
+            if archive_try.status_code != 200:
+                old_archive_url = f"{base_url}/archive/20{self._version[0:2]}/{self._version}_Release/{release_fname}"  # noqa: E501
+                old_archive_try = requests.get(old_archive_url)
+                if old_archive_try.status_code != 200:
+                    msg = (
+                        f"NCIt download failed: tried {src_url}, {archive_url}, and "
+                        f"{old_archive_url}"
+                    )
+                    logger.error(msg)
+                    raise DownloadException(msg)
+                else:
+                    src_url = old_archive_url
+            else:
+                src_url = archive_url
 
-    def _download_data(self):
-        """Download NCI thesaurus source file for loading into normalizer."""
-        logger.info('Downloading NCI Thesaurus...')
-        url = self._SRC_DIR + self._SRC_FNAME
-        zip_path = self._data_path / 'ncit.zip'
-        try:
-            response = requests.get(url, stream=True)
-        except requests.exceptions.RequestException as e:
-            logger.error(f'NCIt download failed: {e}')
-            raise e
-        handle = open(zip_path, "wb")
-        for chunk in response.iter_content(chunk_size=512):
-            if chunk:
-                handle.write(chunk)
-        handle.close()
-        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-            zip_ref.extractall(self._data_path)
-        remove(zip_path)
-        version = self._SRC_DIR.split('/')[-2].split('_')[0]
-        rename(self._data_path / 'Thesaurus.owl', self._data_path / f'ncit_{version}.owl')  # noqa: E501
-        self._version = version
-        logger.info('Finished downloading NCI Thesaurus')
+        self._http_download(src_url, self._src_dir / f"ncit_{self._version}.owl",
+                            handler=self._zip_handler)
+        logger.info("Successfully retrieved source data for NCIt")
 
     def _load_meta(self):
         """Load metadata"""
         metadata = SourceMeta(data_license="CC BY 4.0",
                               data_license_url="https://creativecommons.org/licenses/by/4.0/legalcode",  # noqa F401
-                              version=self._data_file.stem.split('_')[1],
-                              data_url=self._SRC_DIR,
+                              version=self._version,
+                              data_url="https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/",
                               rdp_url='http://reusabledata.org/ncit.html',
                               data_license_attributes={
                                   'non_commercial': False,
diff --git a/disease/etl/omim.py b/disease/etl/omim.py
index c7456bcb..8bcc604a 100644
--- a/disease/etl/omim.py
+++ b/disease/etl/omim.py
@@ -1,46 +1,30 @@
 """Module to load disease data from OMIM."""
 from .base import Base
-from disease import PROJECT_ROOT, DownloadException, logger
+from disease import DownloadException, logger
 from disease.schemas import SourceMeta, SourceName, Disease, NamespacePrefix
-from disease.database import Database
-from pathlib import Path
-from typing import List
 
 
 class OMIM(Base):
     """Gather and load data from OMIM."""
 
-    def __init__(self,
-                 database: Database,
-                 src_url: str = "https://www.omim.org/downloads",
-                 data_path: Path = PROJECT_ROOT / 'data' / 'omim'):
-        """Override base class init method.
-
-        :param therapy.database.Database database: app database instance
-        :param pathlib.Path data_path: path to local OMIM data directory
-        """
-        self._SRC_URL = src_url
-        super().__init__(database=database, data_path=data_path)
-
-    def perform_etl(self) -> List[str]:
-        """Public-facing method to initiate ETL procedures on given data.
-
-        :return: empty list (because OMIM IDs shouldn't be used to construct
-            merged concept groups)
+    def _extract_data(self, use_existing: bool = False):
+        """Override parent extract method to enforce OMIM-specific data file
+        requirements.
+        :param use_existing: technically non-functional, but included to match
+        sibling method signatures. If True, will print warning but otherwise proceed.
         """
+        if not use_existing:
+            logger.warning(
+                "Overruling provided `use_existing` parameter. OMIM data is not "
+                "publicly available - see README for details - and must be manually "
+                f"placed in {self._src_dir.absolute().as_uri()}"
+            )
         try:
-            self._extract_data()
-        except DownloadException:
-            logger.error("OMIM data extraction failed: input file must be "
-                         "manually placed in data directory.")
-            raise DownloadException(f"Could not access OMIM data - see README "
-                                    f"for details. Input data must be "
-                                    f"manually placed in "
-                                    f"{self._data_path.absolute().as_uri()}")
-        self._load_meta()
-        self._transform_data()
-        self.database.flush_batch()
-        return []
+            super()._extract_data(True)
+        except FileNotFoundError:
+            raise FileNotFoundError("Could not locate OMIM data. Per README, OMIM "
+                                    "source files must be manually placed in "
+                                    f"{self._src_dir.absolute().as_uri()}")
 
     def _download_data(self):
         """Download OMIM source data for loading into normalizer."""
@@ -49,9 +33,9 @@ def _download_data(self):
     def _load_meta(self):
         """Load source metadata."""
         metadata = SourceMeta(data_license="custom",
-                              data_license_url="https://omim.org/help/agreement",  # noqa: E501
+                              data_license_url="https://omim.org/help/agreement",
                               version=self._data_file.stem.split('_', 1)[1],
-                              data_url=self._SRC_URL,
+                              data_url="https://www.omim.org/downloads",
                               rdp_url='http://reusabledata.org/omim.html',
                               data_license_attributes={
                                   'non_commercial': False,
diff --git a/disease/etl/oncotree.py b/disease/etl/oncotree.py
index 969ffa70..cf049fd1 100644
--- a/disease/etl/oncotree.py
+++ b/disease/etl/oncotree.py
@@ -1,66 +1,22 @@
 """Module to load disease data from OncoTree."""
+import json
+
 from .base import Base
-from disease import PROJECT_ROOT, logger
+from disease import logger
 from disease.schemas import SourceMeta, SourceName, NamespacePrefix, Disease
-from disease.database import Database
-from pathlib import Path
-from typing import List
-import requests
-import json
 
 
 class OncoTree(Base):
     """Gather and load data from OncoTree."""
 
-    def __init__(self,
-                 database: Database,
-                 src_api_root: str = "http://oncotree.mskcc.org/api/",
-                 data_path: Path = PROJECT_ROOT / 'data' / 'oncotree'):
-        """Override base class init method.
-
-        :param therapy.database.Database database: app database instance
-        :param str src_api_root: root of OncoTree API URL
-        :param pathlib.Path data_path: path to local OncoTree data directory
-        """
-        self._SRC_API_ROOT = src_api_root
-        super().__init__(database=database, data_path=data_path)
-
-    def perform_etl(self) -> List[str]:
-        """Public-facing method to initiate ETL procedures on given data.
-
-        :return: empty list (because OncoTree IDs shouldn't be used to
-            construct merged concept groups)
-        """
-        self._extract_data()
-        self._load_meta()
-        self._transform_data()
-        self.database.flush_batch()
-        return []
-
     def _download_data(self):
         """Download Oncotree source data for loading into normalizer."""
-        logger.info('Downloading OncoTree...')
-        # get version for latest stable release
-        versions_url = f"{self._SRC_API_ROOT}versions"
-        versions = json.loads(requests.get(versions_url).text)
-        latest = [v['release_date'] for v in versions
-                  if v['api_identifier'] == 'oncotree_latest_stable'][0]
-        version = latest.replace('-', '_')
-
-        # download data
-        url = f'{self._SRC_API_ROOT}tumorTypes/tree?version=oncotree_{version}'
-        try:
-            response = requests.get(url, stream=True)
-        except requests.exceptions.RequestException as e:
-            logger.error(f'OncoTree download failed: {e}')
-            raise e
-        filename = self._data_path / f'oncotree_{version}.json'
-        handle = open(filename, 'wb')
-        for chunk in response.iter_content(chunk_size=512):
-            if chunk:
-                handle.write(chunk)
-        self._version = version
-        logger.info('Finished downloading OncoTree')
+        logger.info('Retrieving source data for OncoTree')
+        url_version = self._version.replace("-", "_")
+        url = f'http://oncotree.mskcc.org/api/tumorTypes/tree?version=oncotree_{url_version}'  # noqa: E501
+        output_file = self._src_dir / f"oncotree_{self._version}.json"
+        self._http_download(url, output_file)
+        logger.info('Successfully retrieved source data for OncoTree')
 
     def _load_meta(self):
         """Load metadata"""
diff --git a/disease/etl/utils.py b/disease/etl/utils.py
new file mode 100644
index 00000000..64d9bd39
--- /dev/null
+++ b/disease/etl/utils.py
@@ -0,0 +1,9 @@
+"""Provide useful tools for ETL classes."""
+
+
+class DownloadException(Exception):
+    """Exception for failures relating to source file downloads."""
+
+    def __init__(self, *args, **kwargs):  # noqa: ANN204
+        """Initialize exception."""
+        super().__init__(*args, **kwargs)
diff --git a/disease/query.py b/disease/query.py
index 3a334f51..488b72b6 100644
--- a/disease/query.py
+++ b/disease/query.py
@@ -315,7 +315,7 @@ def _add_merged_meta(self, response: Dict) -> Dict:
         """
         sources_meta = {}
         vod = response['disease_descriptor']
-        ids = [vod['disease_id']] + vod.get('xrefs', [])
+        ids = [vod['disease']] + vod.get('xrefs', [])
         for concept_id in ids:
             prefix = concept_id.split(':')[0]
             src_name = PREFIX_LOOKUP[prefix.lower()]
@@ -337,7 +337,7 @@ def _add_vod(self, response: Dict, record: Dict, query: str,
         vod = {
             'id': f'normalize.disease:{quote(query)}',
             'type': 'DiseaseDescriptor',
-            'disease_id': record['concept_id'],
+            'disease': record['concept_id'],
             'label': record['label'],
             'extensions': [],
         }
diff --git a/disease/schemas.py b/disease/schemas.py
index 6f4a4d3c..20b77764 100644
--- a/disease/schemas.py
+++ b/disease/schemas.py
@@ -338,7 +338,7 @@ def schema_extra(schema: Dict[str, Any],
                 "disease_descriptor": {
                     "id": "normalize:childhood%20leukemia",
                     "type": "DiseaseDescriptor",
-                    "disease_id": "ncit:C4989",
+                    "disease": "ncit:C4989",
                     "label": "Childhood Leukemia",
                     "xrefs": [
                         "mondo:0004355",
diff --git a/disease/version.py b/disease/version.py
index 7da88dce..6429a699 100644
--- a/disease/version.py
+++ b/disease/version.py
@@ -1,2 +1,2 @@
 """Disease normalizer version"""
-__version__ = "0.2.15"
+__version__ = "0.2.16"
diff --git a/dynamodb_revisions/README.md b/dynamodb_revisions/README.md
deleted file mode 100644
index 9964cf02..00000000
--- a/dynamodb_revisions/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# DynamoDB Revisions
-Files in this directory are used to make updates to the `disease_concepts` and/or `disease_metadata` tables in DynamoDB.
-
-## add_omim_other_ids.py
-
-Covers some miscellaneous DB schematic updates:
-
-* Altering merged record concept ID and reference patterns
-* Adding OMIM as an `other_identifier` rather than an `xref`
diff --git a/dynamodb_revisions/add_omim_other_ids.py b/dynamodb_revisions/add_omim_other_ids.py
deleted file mode 100644
index 2177b6a4..00000000
--- a/dynamodb_revisions/add_omim_other_ids.py
+++ /dev/null
@@ -1,151 +0,0 @@
-"""Add v0.2.0 updates to production DynamoDB instance."""
-from disease.database import Database
-from timeit import default_timer as timer
-from disease.etl import Merge
-
-
-db = Database()
-merge = Merge(db)
-
-
-def update_identities():
-    """
-    For all records where item_type='identity':
-     * Move OMIM refs from xrefs to other_ids
-     * Add other_id lookup references
-     * update merge_ref field
-    """
-    last_evaluated_key = None
-
-    while True:
-        if last_evaluated_key:
-            response = db.dynamodb_client.scan(
-                TableName=db.diseases.name,
-                ExclusiveStartKey=last_evaluated_key,
-                FilterExpression='item_type = :item_type and src_name <> :src_name',  # noqa: E501
-                ExpressionAttributeValues={
-                    ':item_type': {'S': 'identity'},
-                    ':src_name': {'S': 'OMIM'}
-                }
-            )
-        else:
-            response = db.dynamodb_client.scan(
-                TableName=db.diseases.name,
-                FilterExpression='item_type = :item_type and src_name <> :src_name',  # noqa: E501
-                ExpressionAttributeValues={
-                    ':item_type': {'S': 'identity'},
-                    ':src_name': {'S': 'OMIM'}
-                }
-            )
-        last_evaluated_key = response.get('LastEvaluatedKey')
-        records = response['Items']
-
-        for record in records:
-            concept_id = record['concept_id']
-
-            # move OMIM refs from xrefs to other_ids
-            update_ids = False
-            other_ids = record.get('other_identifiers', [])
-            xrefs = record.get('xrefs', [])
-            new_xrefs = xrefs[:]
-            for xref in xrefs:
-                if xref.startswith('omim'):
-                    other_ids.append(xref)
-                    new_xrefs.remove(xref)
-                    update_ids = True
-            if update_ids:
-                if new_xrefs:
-                    db.update_record(concept_id, 'xrefs', new_xrefs)
-                else:
-                    key = {
-                        'label_and_type': record['label_and_type'],
-                        'concept_id': record['concept_id']
-                    }
-                    db.diseases.update_item(Key=key,
-                                            UpdateExpression="REMOVE xrefs")
-                db.update_record(concept_id, 'other_identifiers',
-                                 other_ids)
-
-            # add other_id lookup
-            for other_id in other_ids:
-                db.add_ref_record(other_id, concept_id, 'other_id')
-
-            # fix merge ref ID
-            merge_ref = record.get('merge_ref')
-            if merge_ref and '|' in merge_ref:
-                new_merge_ref = merge_ref.split('|')[0]
-                db.update_record(new_merge_ref, concept_id, 'merge_ref')
-
-
-def replace_merged():
-    """Delete + replace existing merged records"""
-    last_evaluated_key = None
-    while True:
-        if last_evaluated_key:
-            response = db.dynamodb_client.scan(
-                TableName=db.diseases.name,
-                ExclusiveStartKey=last_evaluated_key,
-                FilterExpression='item_type = :item_type',
-                ExpressionAttributeValues={
-                    ':item_type': {'S': 'merger'}
-                }
-            )
-        else:
-            response = db.dynamodb_client.scan(
-                TableName=db.diseases.name,
-                FilterExpression='item_type = :item_type',
-                ExpressionAttributeValues={
-                    ':item_type': {'S': 'merger'}
-                }
-            )
-
-        last_evaluated_key = response.get('LastEvaluatedKey')
-        records = response['Items']
-
-        for record in records:
-            concept_id_old = record['concept_id']
-            # delete old record
-            db.batch.delete_item(Key={
-                'label_and_type': record['label_and_type'],
-                'concept_id': concept_id_old
-            })
-
-            # update attributes
-            regenerate = False
-            associated_ids = concept_id_old.split('|')
-            main_id = associated_ids[0]
-            xrefs = record.get('xrefs', [])
-            if xrefs:
-                xrefs_new = xrefs[:]
-            for xref in xrefs:
-                if xref.startswith('omim'):
-                    regenerate = True
-                    associated_ids.append(xref)
-                    xrefs_new.remove(xref)
-                    db.update_record(xref, 'merge_ref', main_id)
-            if regenerate:
-                record, _ = merge._generate_merged_record(associated_ids)
-            else:
-                record['concept_id'] = main_id
-                record['label_and_type'] = f'{main_id.lower()}##merger'
-                if len(associated_ids) > 1:
-                    record['other_ids'] = associated_ids[1:]
-
-            # upload
-            db.batch.put_item(Item=record)
-
-    db.flush_batch()
-
-
-if __name__ == '__main__':
-    # run identity record iteration
-    start = timer()
-    update_identities()
-    end = timer()
-    print(f"Completed non-Mondo updates in {end - start:.5f} seconds")
-
-    # generate merged items + add merge refs to omim
-    start = timer()
-    replace_merged()
-    end = timer()
-    print(f"Completed non-Mondo updates in {end - start:.5f} seconds")
diff --git a/dynamodb_revisions/add_xref_refs.py b/dynamodb_revisions/add_xref_refs.py
deleted file mode 100644
index 258b4519..00000000
--- a/dynamodb_revisions/add_xref_refs.py
+++ /dev/null
@@ -1,95 +0,0 @@
-"""Add xref reference to current concepts and update ICD-O namespace."""
-import sys
-from pathlib import Path
-from timeit import default_timer as timer
-import click
-from boto3.dynamodb.conditions import Attr
-
-PROJECT_ROOT = Path(__file__).resolve().parents[1]
-sys.path.append(f"{PROJECT_ROOT}")
-
-from disease.database import Database  # noqa: E402
-
-
-def update_identity_records():
-    """Add reference for xref attribute and update ICD-O namespace."""
-    db = Database()
-    batch = db.diseases.batch_writer()
-
-    last_evaluated_key = None
-    while True:
-        if last_evaluated_key:
-            response = db.diseases.scan(
-                FilterExpression=Attr('item_type').eq('identity'),
-                ExclusiveStartKey=last_evaluated_key
-            )
-        else:
-            response = db.diseases.scan(
-                FilterExpression=Attr('item_type').eq('identity')
-            )
-        last_evaluated_key = response.get('LastEvaluatedKey')
-
-        records = response['Items']
-        for record in records:
-            xrefs = {xref for xref in record.get('xrefs', [])}
-
-            # update icdo prefix
-            xrefs_updated = {xref.replace('icd.o', 'icdo')
-                             for xref in xrefs}
-            if xrefs != xrefs_updated:
-                db.update_record(record['concept_id'], 'xrefs',
-                                 list(xrefs_updated))
-
-            # add xref lookup
-            xrefs_l = {xref.lower() for xref in xrefs_updated}
-            for xref in xrefs_l:
-                batch.put_item(Item={
-                    'label_and_type': f"{xref}##xref",
-                    'concept_id': record['concept_id'].lower(),
-                    'src_name': record['src_name']
-                })
-
-        if not last_evaluated_key:
-            break
-    db.flush_batch()
-
-
-def update_merged_records():
-    """Update ICD-O namespace in merged records."""
-    db = Database()
-
-    last_evaluated_key = None
-    while True:
-        if last_evaluated_key:
-            response = db.diseases.scan(
-                FilterExpression=Attr('item_type').eq('merger'),
-                ExclusiveStartKey=last_evaluated_key
-            )
-        else:
-            response = db.diseases.scan(
-                FilterExpression=Attr('item_type').eq('merger')
-            )
-        last_evaluated_key = response.get('LastEvaluatedKey')
-
-        records = response['Items']
-        for record in records:
-            xrefs = {xref for xref in record.get('xrefs', [])}
-
-            # update icdo prefix
-            xrefs_updated = {xref.replace('icd.o', 'icdo')
-                             for xref in xrefs}
-            if xrefs != xrefs_updated:
-                db.update_record(record['concept_id'], 'xrefs',
-                                 list(xrefs_updated), 'merger')
-
-        if not last_evaluated_key:
-            break
-
-
-if __name__ == '__main__':
-    click.echo("Adding xref references and updating ICD-O namespace...")
-    start = timer()
-    update_identity_records()
-    update_merged_records()
-    end = timer()
-    click.echo(f"Finished adding xref references in {end - start:.5f}s.")
diff --git a/requirements-dev.txt b/requirements-dev.txt
index a41743cc..28ac2642 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,90 +1,115 @@
 -i https://pypi.org/simple
-anyio==3.6.2 ; python_full_version >= '3.6.2'
-appnope==0.1.3 ; platform_system == 'Darwin'
-asttokens==2.0.8
-attrs==22.1.0 ; python_version >= '3.5'
+anyio==3.6.2; python_full_version >= '3.6.2'
+appnope==0.1.3; platform_system == 'Darwin'
+asttokens==2.1.0
+attrs==22.1.0; python_version >= '3.5'
 backcall==0.2.0
-boto3==1.25.3
-botocore==1.28.3 ; python_version >= '3.7'
-certifi==2022.9.24 ; python_version >= '3.6'
-cfgv==3.3.1 ; python_full_version >= '3.6.1'
-charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
-click==8.1.3 ; python_version >= '3.7'
-contourpy==1.0.5 ; python_version >= '3.7'
+beautifulsoup4==4.11.1; python_version >= '3.6'
+bioregistry[align]==0.6.2; python_version >= '3.7'
+bioversions==0.5.79
+boto3==1.25.5
+botocore==1.28.5; python_version >= '3.7'
+cachier==1.5.4
+certifi==2022.9.24; python_version >= '3.6'
+cfgv==3.3.1; python_full_version >= '3.6.1'
+charset-normalizer==2.1.1; python_version >= '3.6'
+class-resolver==0.3.10
+click==8.1.3
+click-default-group==1.2.2
+contourpy==1.0.6; python_version >= '3.7'
 coverage==6.5.0
 coveralls==3.3.1
-cycler==0.11.0 ; python_version >= '3.6'
-debugpy==1.6.3 ; python_version >= '3.7'
-decorator==5.1.1 ; python_version >= '3.5'
+curies==0.4.0; python_version >= '3.7'
+cycler==0.11.0; python_version >= '3.6'
+dataclasses-json==0.5.7; python_version >= '3.6'
+debugpy==1.6.3; python_version >= '3.7'
+decorator==5.1.1; python_version >= '3.5'
+defusedxml==0.7.1
 -e .
 distlib==0.3.6
 docopt==0.6.2
-entrypoints==0.4 ; python_version >= '3.6'
-exceptiongroup==1.0.0 ; python_version < '3.11'
-executing==1.1.1
-fastapi==0.85.1
-filelock==3.8.0 ; python_version >= '3.7'
+entrypoints==0.4; python_version >= '3.6'
+exceptiongroup==1.0.0; python_version < '3.11'
+executing==1.2.0
+fastapi==0.85.2
+filelock==3.8.0; python_version >= '3.7'
 flake8==5.0.4
 flake8-docstrings==1.6.0
-fonttools==4.38.0 ; python_version >= '3.7'
+fonttools==4.38.0; python_version >= '3.7'
 ga4gh.vrsatile.pydantic==0.0.11
-h11==0.14.0 ; python_version >= '3.7'
-identify==2.5.8 ; python_version >= '3.7'
-idna==3.4 ; python_version >= '3.5'
+h11==0.14.0; python_version >= '3.7'
+identify==2.5.8; python_version >= '3.7'
+idna==3.4; python_version >= '3.5'
 iniconfig==1.1.1
-ipykernel==6.16.2
-ipython==8.5.0 ; python_version >= '3.8'
+ipykernel==6.17.0
+ipython==8.6.0; python_version >= '3.8'
 isodate==0.6.1
-jedi==0.18.1 ; python_version >= '3.6'
-jmespath==1.0.1 ; python_version >= '3.7'
-jupyter-client==7.4.4 ; python_version >= '3.7'
-jupyter-core==4.11.2 ; python_version >= '3.7'
-kiwisolver==1.4.4 ; python_version >= '3.7'
+jedi==0.18.1; python_version >= '3.6'
+jmespath==1.0.1; python_version >= '3.7'
+jupyter-client==7.4.4; python_version >= '3.7'
+jupyter-core==4.11.2; python_version >= '3.7'
+kiwisolver==1.4.4; python_version >= '3.7'
+lxml==4.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+marshmallow==3.18.0; python_version >= '3.7'
+marshmallow-enum==1.5.1
 matplotlib==3.6.1
-matplotlib-inline==0.1.6 ; python_version >= '3.5'
-mccabe==0.7.0 ; python_version >= '3.6'
-nest-asyncio==1.5.6 ; python_version >= '3.5'
-nodeenv==1.7.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
-numpy==1.23.4 ; python_version >= '3.8'
+matplotlib-inline==0.1.6; python_version >= '3.5'
+mccabe==0.7.0; python_version >= '3.6'
+more-click==0.1.1; python_version >= '3.7'
+mypy-extensions==0.4.3
+nest-asyncio==1.5.6; python_version >= '3.5'
+nodeenv==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
+numpy==1.23.4; python_version >= '3.8'
 owlready2==0.39
-packaging==21.3 ; python_version >= '3.6'
-parso==0.8.3 ; python_version >= '3.6'
-pexpect==4.8.0 ; sys_platform != 'win32'
+packaging==21.3; python_version >= '3.6'
+parso==0.8.3; python_version >= '3.6'
+pathtools==0.1.2
+pexpect==4.8.0; sys_platform != 'win32'
 pickleshare==0.7.5
-pillow==9.2.0 ; python_version >= '3.7'
-platformdirs==2.5.2 ; python_version >= '3.7'
-pluggy==1.0.0 ; python_version >= '3.6'
+pillow==9.3.0; python_version >= '3.7'
+platformdirs==2.5.2; python_version >= '3.7'
+pluggy==1.0.0; python_version >= '3.6'
+portalocker==2.6.0; python_version >= '3.5'
 pre-commit==2.20.0
-prompt-toolkit==3.0.31 ; python_full_version >= '3.6.2'
-psutil==5.9.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-ptyprocess==0.7.0
+prompt-toolkit==3.0.31; python_full_version >= '3.6.2'
+psutil==5.9.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+ptyprocess==0.7.0; os_name != 'nt'
 pure-eval==0.2.2
-pycodestyle==2.9.1 ; python_version >= '3.6'
+pycodestyle==2.9.1; python_version >= '3.6'
 pydantic==1.10.2
-pydocstyle==6.1.1 ; python_version >= '3.6'
-pyflakes==2.5.0 ; python_version >= '3.6'
-pygments==2.13.0 ; python_version >= '3.6'
-pyparsing==3.0.9 ; python_full_version >= '3.6.8'
+pydocstyle==6.1.1; python_version >= '3.6'
+pyflakes==2.5.0; python_version >= '3.6'
+pygments==2.13.0; python_version >= '3.6'
+pyparsing==3.0.9; python_full_version >= '3.6.8'
+pystow==0.4.7; python_version >= '3.7'
 pytest==7.2.0
 pytest-cov==4.0.0
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
-pyyaml==6.0 ; python_version >= '3.6'
-pyzmq==24.0.1 ; python_version >= '3.6'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pytrie==0.4.0
+pyyaml==6.0; python_version >= '3.6'
+pyzmq==24.0.1; python_version >= '3.6'
 rdflib==6.2.0
-requests==2.28.1 ; python_version >= '3.7' and python_version < '4'
-s3transfer==0.6.0 ; python_version >= '3.7'
-setuptools==65.5.0 ; python_version >= '3.7'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
-sniffio==1.3.0 ; python_version >= '3.7'
+requests==2.28.1
+requests-ftp==0.3.1
+s3transfer==0.6.0; python_version >= '3.7'
+setuptools==65.5.0; python_version >= '3.7'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sniffio==1.3.0; python_version >= '3.7'
 snowballstemmer==2.2.0
-stack-data==0.5.1
-starlette==0.20.4 ; python_version >= '3.7'
-toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
-tomli==2.0.1 ; python_version < '3.11'
-tornado==6.2 ; python_version >= '3.7'
-traitlets==5.5.0 ; python_version >= '3.7'
-typing-extensions==4.4.0 ; python_version >= '3.7'
-urllib3==1.26.12 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
+sortedcontainers==2.4.0
+soupsieve==2.3.2.post1; python_version >= '3.6'
+stack-data==0.6.0
+starlette==0.20.4; python_version >= '3.7'
+tabulate==0.9.0; python_version >= '3.7'
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+tomli==2.0.1; python_version < '3.11'
+tornado==6.2; python_version >= '3.7'
+tqdm==4.64.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+traitlets==5.5.0; python_version >= '3.7'
+typing-extensions==4.4.0; python_version < '3.10'
+typing-inspect==0.8.0
+urllib3==1.26.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
 uvicorn==0.19.0
-virtualenv==20.16.6 ; python_version >= '3.6'
+virtualenv==20.16.6; python_version >= '3.6'
+watchdog==2.1.9; python_version >= '3.6'
 wcwidth==0.2.5
diff --git a/requirements.txt b/requirements.txt
index 2c150ba4..3542198e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,19 +1,19 @@
 -i https://pypi.org/simple
-anyio==3.6.2 ; python_full_version >= '3.6.2'
-boto3==1.25.3
-botocore==1.28.3 ; python_version >= '3.7'
-click==8.1.3 ; python_version >= '3.7'
-fastapi==0.85.1
+anyio==3.6.2; python_full_version >= '3.6.2'
+boto3==1.25.5
+botocore==1.28.5; python_version >= '3.7'
+click==8.1.3
+fastapi==0.85.2
 ga4gh.vrsatile.pydantic==0.0.11
-h11==0.14.0 ; python_version >= '3.7'
-idna==3.4 ; python_version >= '3.5'
-jmespath==1.0.1 ; python_version >= '3.7'
+h11==0.14.0; python_version >= '3.7'
+idna==3.4; python_version >= '3.5'
+jmespath==1.0.1; python_version >= '3.7'
 pydantic==1.10.2
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
-s3transfer==0.6.0 ; python_version >= '3.7'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
-sniffio==1.3.0 ; python_version >= '3.7'
-starlette==0.20.4 ; python_version >= '3.7'
-typing-extensions==4.4.0 ; python_version >= '3.7'
-urllib3==1.26.12 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+s3transfer==0.6.0; python_version >= '3.7'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+sniffio==1.3.0; python_version >= '3.7'
+starlette==0.20.4; python_version >= '3.7'
+typing-extensions==4.4.0; python_version < '3.10'
+urllib3==1.26.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
 uvicorn==0.19.0
diff --git a/setup.cfg b/setup.cfg
index 1258d7c3..9850a0af 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -29,12 +29,23 @@ install_requires =
     uvicorn
     boto3
     ga4gh.vrsatile.pydantic == 0.0.11
+    click
 
 [options.extras_require]
 dev =
-    click
     owlready2
     rdflib
+    requests
+    bioversions
+    pre-commit
+    flake8
+    flake8-docstrings
+
+test =
+    pytest
+    pytest-cov
+    coveralls
+    coverage
 
 [tool:pytest]
 addopts = --ignore=setup.py --ignore=codebuild/ --ignore=analysis/ --doctest-modules --cov-report term-missing --cov .


Test patch:
diff --git a/tests/unit/data/diseases.json b/tests/unit/data/diseases.json
index 0de64295..7a47511f 100644
--- a/tests/unit/data/diseases.json
+++ b/tests/unit/data/diseases.json
@@ -7,17 +7,17 @@
     "concept_id": "ncit:C2926",
     "label": "Lung Non-Small Cell Carcinoma",
     "aliases": [
+      "Non-Small Cell Carcinoma of Lung",
       "Non Small Cell Lung Cancer NOS",
-      "Non-Small Cell Lung Cancer",
-      "Non-Small Cell Cancer of the Lung",
       "NSCLC",
       "non-small cell lung cancer",
       "Non-Small Cell Carcinoma of the Lung",
-      "Non-Small Cell Cancer of Lung",
+      "Non-Small Cell Cancer of the Lung",
+      "Non-Small Cell Lung Carcinoma",
       "Non-small cell lung cancer, NOS",
-      "Non-Small Cell Carcinoma of Lung",
+      "Non-Small Cell Lung Cancer",
       "NSCLC - Non-Small Cell Lung Cancer",
-      "Non-Small Cell Lung Carcinoma"
+      "Non-Small Cell Cancer of Lung"
     ],
     "associated_with": ["umls:C0007131"]
   },
@@ -103,7 +103,7 @@
       "Neuroblastoma, NOS",
       "neuroblastoma"
     ],
-    "associated_with": ["umls:C0027819", "icdo:9500/3"]
+    "associated_with": ["umls:C2751421", "icdo:9500/3"]
   },
   {
     "label_and_type": "icdo:9500/3##associated_with",
@@ -169,7 +169,6 @@
       "icdo:9500/3",
       "umls:CN205405",
       "efo:0000621",
-      "icd10:C74.9",
       "mesh:D009447"
     ]
   },
@@ -228,7 +227,7 @@
       "Richter's transformation"
     ],
     "xrefs": ["ncit:C35424", "DOID:1703"],
-    "associated_with": ["icd10:C91.1", "umls:C0349631", "gard:0007578"]
+    "associated_with": ["umls:C0349631", "gard:0007578"]
   },
   {
     "label_and_type": "gard:0007578##associated_with",
@@ -266,7 +265,7 @@
     "merge_ref": "ncit:C8091",
     "concept_id": "mondo:0003587",
     "label": "pediatric liposarcoma",
-    "aliases": ["childhood liposarcoma", "liposarcoma"],
+    "aliases": ["childhood liposarcoma"],
     "pediatric_disease": true,
     "xrefs": ["DOID:5695", "ncit:C8091"],
     "associated_with": ["umls:C0279984"],
@@ -336,7 +335,7 @@
     "associated_with": [
       "efo:0000621",
       "gard:7185",
-      "icdo:M9500/3",
+      "icdo:9500/3",
       "mesh:D009447",
       "orphanet:635",
       "umls:C0027819"
@@ -430,23 +429,23 @@
     "src_name": "Mondo",
     "merge_ref": "ncit:c2926",
     "concept_id": "mondo:0005233",
-    "label": "non-small cell lung carcinoma (disease)",
+    "label": "non-small cell lung carcinoma",
     "aliases": [
-      "NSCLC - non-small cell lung cancer",
-      "non-small cell lung carcinoma",
-      "non-small cell carcinoma of lung",
-      "non-small cell carcinoma of the lung",
-      "non-small cell cancer of lung",
-      "non-small cell lung cancer",
-      "non-small cell cancer of the lung",
-      "NSCLC"
+       "non-small cell cancer of lung",
+       "non-small cell carcinoma of the lung",
+       "NSCLC",
+       "non-small cell lung cancer",
+       "non-small cell lung carcinoma (disease)",
+       "NSCLC - non-small cell lung cancer",
+       "non-small cell carcinoma of lung",
+       "non-small cell cancer of the lung"
     ],
     "xrefs": ["ncit:C2926", "oncotree:NSCLC", "DOID:3908"],
     "associated_with": [
       "mesh:D002289",
       "umls:C0007131",
       "efo:0003060",
-      "icd10:C34",
+      "orphanet:488201",
       "kegg.disease:05223",
       "HP:0030358"
     ]
@@ -641,6 +640,11 @@
       "kegg.disease:05223",
       "efo:0003060",
       "mesh:D002289"
+    ],
+    "aliases": [
+      "non-small cell lung carcinoma",
+      "Non-small cell lung cancer",
+      "NSCLC"
     ]
   },
   {
@@ -672,13 +676,12 @@
       "efo:0000621",
       "gard:0007185",
       "gard:7185",
-      "icd10:C74.9",
       "icdo:9500/3",
-      "icdo:M9500/3",
       "mesh:D009447",
       "meddra:10029260",
       "nifstd:birnlex_12631",
       "orphanet:635",
+      "umls:C2751421",
       "umls:CN205405"
     ]
   },
@@ -712,7 +715,7 @@
       "mesh:D002289",
       "umls:C0007131",
       "efo:0003060",
-      "icd10:C34",
+      "orphanet:488201",
       "kegg.disease:05223",
       "HP:0030358"
     ]
@@ -733,7 +736,6 @@
     ],
     "associated_with": [
       "umls:C0349631",
-      "icd10:C91.1",
       "gard:0007578",
       "gard:7578",
       "icd10.cm:C91.1"
@@ -771,9 +773,9 @@
     "merge_ref": "mondo:0010648",
     "label": "major affective disorder 2",
     "concept_id": "mondo:0010648",
-    "aliases": ["MAFD2"],
-    "xrefs": ["DOID:0080221"],
-    "associated_with": ["mesh:C564108", "omim:309200"]
+    "aliases": ["MAFD2", "major affective disorder 2, X-linked dominant"],
+    "xrefs": ["DOID:0080221", "omim:309200"],
+    "associated_with": ["mesh:C564108"]
   },
   {
     "label_and_type": "major affective disorder 2##label",
@@ -792,7 +794,7 @@
     "item_type": "merger",
     "concept_id": "mondo:0010648",
     "label": "major affective disorder 2",
-    "aliases": ["MDI","MDX","MANIC-DEPRESSIVE ILLNESS","BPAD","MAFD2","BIPOLAR AFFECTIVE DISORDER","MANIC-DEPRESSIVE PSYCHOSIS, X-LINKED"],
+    "aliases": ["MDI","MDX","MANIC-DEPRESSIVE ILLNESS","BPAD","MAFD2","BIPOLAR AFFECTIVE DISORDER","MANIC-DEPRESSIVE PSYCHOSIS, X-LINKED","major affective disorder 2, X-linked dominant"],
     "xrefs": ["omim:309200"],
     "associated_with": ["mesh:C564108"]
   },
diff --git a/tests/unit/data/metadata.json b/tests/unit/data/metadata.json
index b940a43c..f354eca1 100644
--- a/tests/unit/data/metadata.json
+++ b/tests/unit/data/metadata.json
@@ -3,8 +3,8 @@
     "src_name": "NCIt",
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
-    "version": "21.01d",
-    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/21.01d_Release/",
+    "version": "22.09d",
+    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/",
     "rdp_url": "http://reusabledata.org/ncit.html",
     "data_license_attributes": {
       "non_commercial": false,
@@ -16,7 +16,7 @@
     "src_name": "Mondo",
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
-    "version": "20210129",
+    "version": "2022-10-11",
     "data_url": "https://mondo.monarchinitiative.org/pages/download/",
     "rdp_url": "http://reusabledata.org/monarch.html",
     "data_license_attributes": {
@@ -42,7 +42,7 @@
     "src_name": "OncoTree",
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
-    "version": "2020_10_01",
+    "version": "2021-11-02",
     "data_url": "http://oncotree.mskcc.org/#/home?tab=api",
     "rdp_url": null,
     "data_license_attributes": {
diff --git a/tests/unit/test_do.py b/tests/unit/test_do.py
index ae415d36..5a636985 100644
--- a/tests/unit/test_do.py
+++ b/tests/unit/test_do.py
@@ -31,7 +31,7 @@ def neuroblastoma():
         "associated_with": [
             "efo:0000621",
             "gard:7185",
-            "icdo:M9500/3",
+            "icdo:9500/3",
             "mesh:D009447",
             "orphanet:635",
             "umls:C0027819",
diff --git a/tests/unit/test_merge.py b/tests/unit/test_merge.py
index 385c479c..9e4814e7 100644
--- a/tests/unit/test_merge.py
+++ b/tests/unit/test_merge.py
@@ -51,12 +51,11 @@ def neuroblastoma():
         ],
         "associated_with": [
             "umls:C0027819",
+            "umls:C2751421",
             "icdo:9500/3",
             "efo:0000621",
             "gard:7185",
             "gard:0007185",
-            "icd10:C74.9",
-            "icdo:M9500/3",
             "mesh:D009447",
             "meddra:10029260",
             "nifstd:birnlex_12631",
@@ -76,32 +75,33 @@ def lnscc():
         "xrefs": ["mondo:0005233", "oncotree:NSCLC", "DOID:3908"],
         "label": "Lung Non-Small Cell Carcinoma",
         "aliases": [
-            "Non Small Cell Lung Cancer NOS",
-            "Non-Small Cell Lung Cancer",
-            "Non-Small Cell Cancer of the Lung",
+            "NSCLC - Non-Small Cell Lung Cancer",
+            "NSCLC - non-small cell lung cancer",
             "NSCLC",
-            "non-small cell lung cancer",
-            "Non-Small Cell Carcinoma of the Lung",
+            "Non Small Cell Lung Cancer NOS",
             "Non-Small Cell Cancer of Lung",
-            "Non-small cell lung cancer, NOS",
+            "Non-Small Cell Cancer of the Lung",
             "Non-Small Cell Carcinoma of Lung",
-            "NSCLC - Non-Small Cell Lung Cancer",
+            "Non-Small Cell Carcinoma of the Lung",
+            "Non-Small Cell Lung Cancer",
             "Non-Small Cell Lung Carcinoma",
-            "NSCLC - non-small cell lung cancer",
-            "non-small cell lung carcinoma",
-            "non-small cell carcinoma of lung",
-            "non-small cell carcinoma of the lung",
+            "Non-small cell lung cancer",
+            "Non-small cell lung cancer, NOS",
             "non-small cell cancer of lung",
             "non-small cell cancer of the lung",
+            "non-small cell carcinoma of lung",
+            "non-small cell carcinoma of the lung",
+            "non-small cell lung cancer",
+            "non-small cell lung carcinoma (disease)",
+            "non-small cell lung carcinoma",
         ],
         "associated_with": [
             "umls:C0007131",
             "mesh:D002289",
-            "umls:C0007131",
             "efo:0003060",
-            "icd10:C34",
             "kegg.disease:05223",
-            "HP:0030358"
+            "HP:0030358",
+            "orphanet:488201"
         ],
         "item_type": "merger"
     }
@@ -125,7 +125,6 @@ def richter():
         ],
         "associated_with": [
             "umls:C0349631",
-            "icd10:C91.1",
             "gard:0007578",
             "gard:7578",
             "icd10.cm:C91.1"
@@ -146,7 +145,6 @@ def ped_liposarcoma():
             "Liposarcoma",
             "Pediatric Liposarcoma",
             "childhood liposarcoma",
-            "liposarcoma",
             "pediatric liposarcoma"
         ],
         "associated_with": ["umls:C0279984"],
@@ -180,11 +178,17 @@ def mafd2():
         "concept_id": "mondo:0010648",
         "label": "major affective disorder 2",
         "aliases": [
-            "MAFD2"
+          "MANIC-DEPRESSIVE ILLNESS",
+          "MDX",
+          "MDI",
+          "MANIC-DEPRESSIVE PSYCHOSIS, X-LINKED",
+          "BPAD",
+          "BIPOLAR AFFECTIVE DISORDER",
+          "MAFD2",
+          "major affective disorder 2, X-linked dominant"
         ],
         "associated_with": [
-            "mesh:C564108",
-            "omim:309200"
+            "mesh:C564108"
         ]
     }
 
diff --git a/tests/unit/test_mondo.py b/tests/unit/test_mondo.py
index 4635df61..9cbc5b95 100644
--- a/tests/unit/test_mondo.py
+++ b/tests/unit/test_mondo.py
@@ -45,7 +45,6 @@ def neuroblastoma():
             "icdo:9500/3",
             "umls:CN205405",
             "efo:0000621",
-            "icd10:C74.9",
             "mesh:D009447"
         ],
         "pediatric_disease": None,
@@ -65,7 +64,6 @@ def richter_syndrome():
         ],
         "xrefs": ["ncit:C35424", "DOID:1703"],
         "associated_with": [
-            "icd10:C91.1",
             "umls:C0349631",
             "gard:0007578",
         ],
@@ -83,7 +81,6 @@ def pediatric_liposarcoma():
         "label": "pediatric liposarcoma",
         "aliases": [
             "childhood liposarcoma",
-            "liposarcoma"
         ],
         "xrefs": ["DOID:5695", "ncit:C8091"],
         "associated_with": [
@@ -113,10 +110,10 @@ def nsclc():
     """Construct a test fixture for non small cell lung cancer."""
     return Disease(**{
         "concept_id": "mondo:0005233",
-        "label": "non-small cell lung carcinoma (disease)",
+        "label": "non-small cell lung carcinoma",
         "aliases": [
             "NSCLC - non-small cell lung cancer",
-            "non-small cell lung carcinoma",
+            "non-small cell lung carcinoma (disease)",
             "non-small cell carcinoma of lung",
             "non-small cell carcinoma of the lung",
             "non-small cell cancer of lung",
@@ -128,10 +125,10 @@ def nsclc():
         "associated_with": [
             "mesh:D002289",
             "umls:C0007131",
-            "icd10:C34",
             "efo:0003060",
             "kegg.disease:05223",
-            "HP:0030358"
+            "HP:0030358",
+            "orphanet:488201"
         ]
     })
 
@@ -300,7 +297,7 @@ def test_meta(mondo):
     assert response.source_meta_.data_license == 'CC BY 4.0'
     assert response.source_meta_.data_license_url == \
         'https://creativecommons.org/licenses/by/4.0/legalcode'
-    assert response.source_meta_.version == '20210129'
+    assert response.source_meta_.version == "2022-10-11"
     assert response.source_meta_.data_url == \
         'https://mondo.monarchinitiative.org/pages/download/'
     assert response.source_meta_.rdp_url == 'http://reusabledata.org/monarch.html'  # noqa: E501
diff --git a/tests/unit/test_ncit.py b/tests/unit/test_ncit.py
index ad3a2db7..df892a0a 100644
--- a/tests/unit/test_ncit.py
+++ b/tests/unit/test_ncit.py
@@ -37,7 +37,7 @@ def neuroblastoma():
             "neuroblastoma"
         ],
         "xrefs": [],
-        "associated_with": ["umls:C0027819", "icdo:9500/3"],
+        "associated_with": ["umls:C2751421", "icdo:9500/3"],
         "src_name": "NCIt"
     })
 
@@ -183,10 +183,10 @@ def test_meta(ncit):
     assert response.source_meta_.data_license == 'CC BY 4.0'
     assert response.source_meta_.data_license_url == \
         'https://creativecommons.org/licenses/by/4.0/legalcode'
-    assert response.source_meta_.version == '21.01d'
+    assert response.source_meta_.version == '22.09d'
     assert response.source_meta_.data_url == \
-        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/21.01d_Release/"  # noqa: E501
-    assert response.source_meta_.rdp_url == 'http://reusabledata.org/ncit.html'  # noqa: E501
+        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/"
+    assert response.source_meta_.rdp_url == 'http://reusabledata.org/ncit.html'
     assert response.source_meta_.data_license_attributes == {
         "non_commercial": False,
         "share_alike": False,
diff --git a/tests/unit/test_oncotree.py b/tests/unit/test_oncotree.py
index 66102bd5..a3494cca 100644
--- a/tests/unit/test_oncotree.py
+++ b/tests/unit/test_oncotree.py
@@ -139,7 +139,7 @@ def test_meta(oncotree):
     assert response.source_meta_.data_license == 'CC BY 4.0'
     assert response.source_meta_.data_license_url == \
         'https://creativecommons.org/licenses/by/4.0/legalcode'
-    assert response.source_meta_.version == '2020_10_01'
+    assert response.source_meta_.version == "2021-11-02"
     assert response.source_meta_.data_url == \
         'http://oncotree.mskcc.org/#/home?tab=api'
     assert response.source_meta_.rdp_url is None
diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py
index b344695d..e542385d 100644
--- a/tests/unit/test_query.py
+++ b/tests/unit/test_query.py
@@ -31,7 +31,7 @@ def neuroblastoma():
     return ValueObjectDescriptor(**{
         "id": "normalize.disease:Neuroblastoma",
         "type": "DiseaseDescriptor",
-        "disease_id": "ncit:C3270",
+        "disease": "ncit:C3270",
         "label": "Neuroblastoma",
         "xrefs": [
             "mondo:0005072",
@@ -60,14 +60,13 @@ def neuroblastoma():
                     "efo:0000621",
                     "gard:7185",
                     "gard:0007185",
-                    "icd10:C74.9",
                     "icdo:9500/3",
-                    "icdo:M9500/3",
                     "mesh:D009447",
                     "meddra:10029260",
                     "nifstd:birnlex_12631",
                     "orphanet:635",
-                    "umls:CN205405"
+                    "umls:CN205405",
+                    "umls:C2751421"
                 ]
             }
         ]
@@ -80,7 +79,7 @@ def skin_myo():
     return ValueObjectDescriptor(**{
         "id": "normalize.disease:Skin Myoepithelioma",
         "type": "DiseaseDescriptor",
-        "disease_id": "ncit:C167370",
+        "disease": "ncit:C167370",
         "label": "Skin Myoepithelioma",
         "alternate_labels": ["Cutaneous Myoepithelioma"],
     })
@@ -94,7 +93,7 @@ def mafd2():
     return ValueObjectDescriptor(**{
         "id": "normalize.disease:MAFD2",
         "type": "DiseaseDescriptor",
-        "disease_id": "mondo:0010648",
+        "disease": "mondo:0010648",
         "label": "major affective disorder 2",
         "alternate_labels": [
             "MAFD2",
@@ -104,7 +103,8 @@ def mafd2():
             "BPAD",
             "MAFD2",
             "BIPOLAR AFFECTIVE DISORDER",
-            "MANIC-DEPRESSIVE PSYCHOSIS, X-LINKED"
+            "MANIC-DEPRESSIVE PSYCHOSIS, X-LINKED",
+            "major affective disorder 2, X-linked dominant"
         ],
         "xrefs": [
             "omim:309200"
@@ -126,7 +126,7 @@ def compare_vod(actual, fixture):
     actual = actual.disease_descriptor
     assert actual.id == fixture.id
     assert actual.type == fixture.type
-    assert actual.disease_id == fixture.disease_id
+    assert actual.disease == fixture.disease
     assert actual.label == fixture.label
 
     assert (actual.xrefs is not None) == (fixture.xrefs is not None)