Repository: cancervariants/therapy-normalization (language: python)

Issue:
NCIt Download Error
When running `python3 -m therapy.cli --normalizer="ncit" --dev`, I get a` zipfile.BadZipFile: File is not a zip file`. We should fix this to be able to use NCIt.
Change default db endpoint to localhost
Closes #112 


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 92f13999..a4c05e2a 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ for direction on installing pipenv in your compute environment.
  
 Once installed, from the project root dir, just run:
 
-```shell script
+```commandline
 pipenv sync
 ```
 
@@ -34,7 +34,7 @@ This ensures:
 
 Before first commit run:
 
-```
+```commandline
 pre-commit install
 ```
 
@@ -43,11 +43,11 @@ pre-commit install
 
 Running unit tests is as easy as pytest.
 
-```
+```commandline
 pipenv run pytest
 ```
 
-### Updating the gene normalization database
+### Updating the therapy normalization database
 
 Before you use the CLI to update the database, run the following in a separate terminal to start DynamoDB on `port 8000`:
 
@@ -58,11 +58,6 @@ java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb
 To change the port, simply add `-port value`.
 
 ### Setting Environment Variables
-If you have cloned the repo, you must set an environmental variable. If you have installed via pip, you can ignore this step.
-```shell script
-export DEV="true"
-```
-
 RxNorm requires a UMLS license, which you can register for one [here](https://www.nlm.nih.gov/research/umls/index.html).
 You must set the `RxNORM_API_KEY` environment variable to your API key. This can be found in the [UTS 'My Profile' area](https://uts.nlm.nih.gov/uts/profile) after singing in.
 ```shell script
@@ -76,7 +71,7 @@ To update one source, simply set `--normalizer` to the source you wish to update
 
 From the project root, run the following to update the ChEMBL source:
 
-```
+```commandline
 python3 -m therapy.cli --normalizer="chembl"
 ```
 
@@ -88,31 +83,30 @@ To update all sources, use the `--update_all` flag.
 
 From the project root, run the following to update all sources:
 
-```
+```commandline
 python3 -m therapy.cli --update_all
 ```
 
 #### Specifying the database URL endpoint
+The default URL endpoint is `http://localhost:8000`.
+There are two different ways to specify the database URL endpoint.
 
-To specify the database URL endpoint, simply set `--db_url` to the endpoint you want to use. You must also include `--normalizer` or `--update_all`.
-
-From the project root, run the following to update all sources from the database with URL endpoint `http://localhost:8001`:
-
-```
+The first way is to set the `--db_url` flag to the URL endpoint.
+```commandline
 python3 -m therapy.cli --update_all --db_url="http://localhost:8001"
 ```
 
-From the project root, run the following to update all sources from the database with URL endpoint `http://localhost:8000`:
-```
-python3 -m therapy.cli --update_all --dev
+The second way is to set the `THERAPY_NORM_DB_URL` to the URL endpoint.
+```commandline
+export THERAPY_NORM_DB_URL="http://localhost:8001"
+python3 -m therapy.cli --update_all
 ```
 
-
 ### Starting the therapy normalization service
 
 From the project root, run the following:
 
-```shell script
+```commandline
  uvicorn therapy.main:app --reload
 ```
 
diff --git a/therapy/cli.py b/therapy/cli.py
index 0ab9a7cd..f9a696f6 100644
--- a/therapy/cli.py
+++ b/therapy/cli.py
@@ -2,11 +2,11 @@
 import click
 from botocore.exceptions import ClientError
 from therapy.etl import ChEMBL, Wikidata, DrugBank, NCIt, ChemIDplus, RxNorm
+from therapy.etl.merge import Merge
 from therapy.schemas import SourceName
 from timeit import default_timer as timer
 from therapy.database import Database
 from boto3.dynamodb.conditions import Key
-import sys
 from os import environ
 
 
@@ -19,9 +19,9 @@ class CLI:
         help="The normalizer(s) you wish to update separated by spaces."
     )
     @click.option(
-        '--dev',
+        '--prod',
         is_flag=True,
-        help="Working in development environment on localhost port 8000."
+        help="Working in production environment."
     )
     @click.option(
         '--db_url',
@@ -32,7 +32,14 @@ class CLI:
         is_flag=True,
         help='Update all normalizer sources.'
     )
-    def update_normalizer_db(normalizer, dev, db_url, update_all):
+    @click.option(
+        '--update_merged',
+        is_flag=True,
+        help='Update concepts for normalize endpoint. Must select either '
+             '--update_all or include Mondo as a normalizer source argument.'
+    )
+    def update_normalizer_db(normalizer, prod, db_url, update_all,
+                             update_merged):
         """Update select normalizer source(s) in the therapy database."""
         sources = {
             'chembl': ChEMBL,
@@ -43,25 +50,21 @@ def update_normalizer_db(normalizer, dev, db_url, update_all):
             'rxnorm': RxNorm,
         }
 
-        if dev:
-            db: Database = Database(db_url='http://localhost:8000')
-        elif db_url:
-            db: Database = Database(db_url=db_url)
-        elif 'THERAPY_NORM_DB_URL' in environ.keys():
-            # environment variable will be picked up by DB instance
+        if prod:
+            environ['THERAPY_NORM_PROD'] = "TRUE"
             db: Database = Database()
         else:
-            if click.confirm("Are you sure you want to update"
-                             " the production database?", default=False):
-                click.echo("Updating production db...")
-                db: Database = Database()
+            if db_url:
+                endpoint_url = db_url
+            elif 'THERAPY_NORM_DB_URL' in environ.keys():
+                endpoint_url = environ['THERAPY_NORM_DB_URL']
             else:
-                click.echo("Exiting CLI.")
-                sys.exit()
+                endpoint_url = 'http://localhost:8000'
+            db: Database = Database(db_url=endpoint_url)
 
         if update_all:
             normalizers = list(src for src in sources)
-            CLI()._update_normalizers(normalizers, sources, db)
+            CLI()._update_normalizers(normalizers, sources, db, update_merged)
         elif not normalizer:
             CLI()._help_msg()
         else:
@@ -75,7 +78,7 @@ def update_normalizer_db(normalizer, dev, db_url, update_all):
             if len(non_sources) != 0:
                 raise Exception(f"Not valid source(s): {non_sources}")
 
-            CLI()._update_normalizers(normalizers, sources, db)
+            CLI()._update_normalizers(normalizers, sources, db, update_merged)
 
     def _help_msg(self):
         """Display help message."""
@@ -89,8 +92,9 @@ def _check_norm_srcs_match(self, sources, normalizers):
         """Check that entered normalizers are actual sources."""
         return set(normalizers) - {src for src in sources}
 
-    def _update_normalizers(self, normalizers, sources, db):
+    def _update_normalizers(self, normalizers, sources, db, update_merged):
         """Update selected normalizer sources."""
+        processed_ids = list()
         for n in normalizers:
             click.echo(f"\nDeleting {n}...")
             start_delete = timer()
@@ -102,13 +106,26 @@ def _update_normalizers(self, normalizers, sources, db):
             click.echo(f"Loading {n}...")
             start_load = timer()
             source = sources[n](database=db)
-            source.perform_etl()
+            # TODO: Replace with constant once issue-90 merged
+            if n not in ['chembl', 'drugbank']:
+                processed_ids += source.perform_etl()
+            else:
+                source.perform_etl()
             end_load = timer()
             load_time = end_load - start_load
             click.echo(f"Loaded {n} in {load_time:.5f} seconds.")
             click.echo(f"Total time for {n}: "
                        f"{(delete_time + load_time):.5f} seconds.")
 
+        if update_merged and processed_ids:
+            click.echo("Generating merged concepts...")
+            start_merge = timer()
+            merge = Merge(database=db)
+            merge.create_merged_concepts(processed_ids)
+            end_merge = timer()
+            click.echo(f"Merged concept generation completed in"
+                       f" {(end_merge - start_merge):.5f} seconds.")
+
     def _delete_data(self, source, database):
         # Delete source's metadata
         try:
diff --git a/therapy/database.py b/therapy/database.py
index 9b7b8776..adaa6020 100644
--- a/therapy/database.py
+++ b/therapy/database.py
@@ -6,6 +6,8 @@
 from os import environ
 from typing import Optional, Dict, List, Any
 import logging
+import click
+import sys
 
 logger = logging.getLogger('therapy')
 logger.setLevel(logging.DEBUG)
@@ -26,25 +28,37 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'):
         :param str db_url: database endpoint URL to connect to
         :param str region_name: AWS region name to use
         """
-        if db_url:
+        if 'THERAPY_NORM_PROD' in environ.keys():
             boto_params = {
-                'region_name': region_name,
-                'endpoint_url': db_url
-            }
-        elif 'THERAPY_NORM_DB_URL' in environ.keys():
-            boto_params = {
-                'region_name': region_name,
-                'endpoint_url': environ['THERAPY_NORM_DB_URL']
+                'region_name': region_name
             }
+            if 'THERAPY_NORM_EB_PROD' not in environ.keys():
+                # EB Instance should not have to confirm.
+                # This is used only for using production via CLI
+                if click.confirm("Are you sure you want to use the "
+                                 "production database?", default=False):
+                    click.echo("***PRODUCTION DATABASE IN USE***")
+                else:
+                    click.echo("Exiting.")
+                    sys.exit()
         else:
+            if db_url:
+                endpoint_url = db_url
+            elif 'THERAPY_NORM_DB_URL' in environ.keys():
+                endpoint_url = environ['THERAPY_NORM_DB_URL']
+            else:
+                endpoint_url = 'http://localhost:8000'
+            click.echo(f"***Using Database Endpoint: {endpoint_url}***")
             boto_params = {
-                'region_name': region_name
+                'region_name': region_name,
+                'endpoint_url': endpoint_url
             }
+
         self.dynamodb = boto3.resource('dynamodb', **boto_params)
         self.dynamodb_client = boto3.client('dynamodb', **boto_params)
 
-        # create tables if nonexistant if not connecting to remote database
-        if db_url or 'THERAPY_NORM_DB_URL' in environ.keys():
+        # create tables if nonexistent if not connecting to remote database
+        if 'THERAPY_NORM_PROD' not in environ.keys():
             existing_tables = self.dynamodb_client.list_tables()['TableNames']
             self.create_therapies_table(existing_tables)
             self.create_meta_data_table(existing_tables)
diff --git a/therapy/etl/chemidplus.py b/therapy/etl/chemidplus.py
index 73e2c68c..fd65062a 100644
--- a/therapy/etl/chemidplus.py
+++ b/therapy/etl/chemidplus.py
@@ -104,7 +104,7 @@ def _get_file(self, data_dir):
         self._download_data(self._data_path)
         dir_files = list(data_dir.iterdir())
         return sorted([f for f in dir_files
-                       if f.names.startswith('chemidplus')])
+                       if f.name.startswith('chemidplus')])
 
     def _transform_data(self):
         """Open dataset and prepare for loading into database."""
diff --git a/therapy/etl/ncit.py b/therapy/etl/ncit.py
index 10eb33b8..63352c87 100644
--- a/therapy/etl/ncit.py
+++ b/therapy/etl/ncit.py
@@ -27,7 +27,7 @@ class NCIt(Base):
 
     def __init__(self,
                  database: Database,
-                 src_dir: str = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/",  # noqa F401
+                 src_dir: str = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/",  # noqa F401
                  src_fname: str = "Thesaurus_20.09d.OWL.zip",
                  data_path: Path = PROJECT_ROOT / 'data' / 'ncit',
                  chemidplus_path: Path = PROJECT_ROOT / 'data' / 'chemidplus'):
@@ -66,6 +66,7 @@ def _download_data(self):
         for chunk in response.iter_content(chunk_size=512):
             if chunk:
                 handle.write(chunk)
+        handle.close()
         with zipfile.ZipFile(zip_path, 'r') as zip_ref:
             zip_ref.extractall(self._data_path)
         remove(zip_path)
diff --git a/therapy/etl/rxnorm.py b/therapy/etl/rxnorm.py
index 62943d09..2ac81dbd 100644
--- a/therapy/etl/rxnorm.py
+++ b/therapy/etl/rxnorm.py
@@ -114,24 +114,18 @@ def _download_data(self, rxn_dir):
             uri = 'https://download.nlm.nih.gov/umls/kss/' \
                   'rxnorm/RxNorm_full_current.zip'
 
-            if 'DEV' not in environ:
-                RXNORM_PATH = str(Path(therapy.__file__).resolve().parents[0] / 'data' / 'rxnorm' / 'RxNorm_full_current.zip')  # noqa: E501
-                environ['RXNORM_PATH'] = RXNORM_PATH
-                zip_path = RXNORM_PATH
+            rxnorm_path = str(Path(therapy.__file__).resolve().parents[0] / 'data' / 'rxnorm' / 'RxNorm_full_current.zip')  # noqa: E501
+            environ['RXNORM_PATH'] = rxnorm_path
 
             # Source:
             # https://documentation.uts.nlm.nih.gov/automating-downloads.html
             subprocess.call(['bash', f'{PROJECT_ROOT}/etl/'
                                      f'rxnorm_download.sh', uri])
 
-            if 'DEV' in environ:
-                zip_path = rxn_dir / 'RxNorm_full_current.zip'
-                shutil.move(PROJECT_ROOT / 'RxNorm_full_current.zip', zip_path)
-
-            with zipfile.ZipFile(zip_path, 'r') as zf:
+            with zipfile.ZipFile(rxnorm_path, 'r') as zf:
                 zf.extractall(rxn_dir)
 
-            remove(zip_path)
+            remove(rxnorm_path)
             shutil.rmtree(rxn_dir / 'prescribe')
             shutil.rmtree(rxn_dir / 'scripts')
 


Test patch:
diff --git a/tests/unit/data/metadata.json b/tests/unit/data/metadata.json
index eeee3861..e032f40c 100644
--- a/tests/unit/data/metadata.json
+++ b/tests/unit/data/metadata.json
@@ -28,7 +28,7 @@
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
     "version": "20.09d",
-    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/",
+    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/",
     "rdp_url": "http://reusabledata.org/ncit.html",
     "data_license_attributes": {
       "non_commercial": false,
diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py
index 2a27ce76..c974f991 100644
--- a/tests/unit/test_database.py
+++ b/tests/unit/test_database.py
@@ -11,7 +11,7 @@ def db():
     """Create a DynamoDB test fixture."""
     class DB:
         def __init__(self):
-            self.db = Database()
+            self.db = Database(db_url=os.environ['THERAPY_NORM_DB_URL'])
             if os.environ.get('TEST') is not None:
                 self.load_test_data()
 
@@ -32,11 +32,11 @@ def load_test_data(self):
                         batch.put_item(Item=m)
                 f.close()
 
-    return DB()
+    return DB().db
 
 
 def test_tables_created(db):
     """Check that therapy_concepts and therapy_metadata are created."""
-    existing_tables = db.db.dynamodb_client.list_tables()['TableNames']
+    existing_tables = db.dynamodb_client.list_tables()['TableNames']
     assert 'therapy_concepts' in existing_tables
     assert 'therapy_metadata' in existing_tables
diff --git a/tests/unit/test_ncit_source.py b/tests/unit/test_ncit_source.py
index 415f2375..0e06fddb 100644
--- a/tests/unit/test_ncit_source.py
+++ b/tests/unit/test_ncit_source.py
@@ -392,7 +392,7 @@ def test_meta_info(voglibose, ncit):
         'https://creativecommons.org/licenses/by/4.0/legalcode'
     assert normalizer_response['meta_'].version == '20.09d'
     assert normalizer_response['meta_'].data_url == \
-        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/"  # noqa: E501
+        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/"  # noqa: E501
     assert normalizer_response['meta_'].rdp_url == 'http://reusabledata.org/ncit.html'  # noqa: E501
     assert normalizer_response['meta_'].data_license_attributes == {
         "non_commercial": False,