Repository: cancervariants/therapy-normalization (language: python)

Issue:
pypi package errors
- `rxnorm_download.sh` was not in the package_data 
- Chemidplus was missing a parameter in its download method
- Wikidata uses `wikibaseintegrator` for the sparql query and `ItemEngine` no longer has the method `execute_sparql_query `
Issue 110
Closes #107 , #110 


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index fd6d8058..a4c05e2a 100644
--- a/README.md
+++ b/README.md
@@ -11,13 +11,13 @@ for direction on installing pipenv in your compute environment.
  
 Once installed, from the project root dir, just run:
 
-```shell script
+```commandline
 pipenv sync
 ```
 
-### Data files
-For now, data files supporting the normalizers need to be obtained independently.
-See issue #7 for progress on updaters.
+### Deploying DynamoDB Locally
+
+We use Amazon DynamoDB for our database. To deploy locally, follow [these instructions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html).
 
 ### Init coding style tests
 
@@ -34,7 +34,7 @@ This ensures:
 
 Before first commit run:
 
-```
+```commandline
 pre-commit install
 ```
 
@@ -43,18 +43,73 @@ pre-commit install
 
 Running unit tests is as easy as pytest.
 
-```
+```commandline
 pipenv run pytest
 ```
 
+### Updating the therapy normalization database
+
+Before you use the CLI to update the database, run the following in a separate terminal to start DynamoDB on `port 8000`:
+
+```
+java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb
+```
+
+To change the port, simply add `-port value`.
+
+### Setting Environment Variables
+RxNorm requires a UMLS license, which you can register for one [here](https://www.nlm.nih.gov/research/umls/index.html).
+You must set the `RxNORM_API_KEY` environment variable to your API key. This can be found in the [UTS 'My Profile' area](https://uts.nlm.nih.gov/uts/profile) after singing in.
+```shell script
+export RXNORM_API_KEY={rxnorm_api_key}
+``` 
+
+#### Update source(s)
+The sources we currently use are: ChEMBL, NCIt, DrugBank, RxNorm, ChemIDplus, and Wikidata.
+
+To update one source, simply set `--normalizer` to the source you wish to update. 
+
+From the project root, run the following to update the ChEMBL source:
+
+```commandline
+python3 -m therapy.cli --normalizer="chembl"
+```
+
+To update multiple sources, you can use the `normalizer` flag with the source names separated by spaces.
+
+#### Update all sources
+
+To update all sources, use the `--update_all` flag. 
+
+From the project root, run the following to update all sources:
+
+```commandline
+python3 -m therapy.cli --update_all
+```
+
+#### Specifying the database URL endpoint
+The default URL endpoint is `http://localhost:8000`.
+There are two different ways to specify the database URL endpoint.
+
+The first way is to set the `--db_url` flag to the URL endpoint.
+```commandline
+python3 -m therapy.cli --update_all --db_url="http://localhost:8001"
+```
+
+The second way is to set the `THERAPY_NORM_DB_URL` to the URL endpoint.
+```commandline
+export THERAPY_NORM_DB_URL="http://localhost:8001"
+python3 -m therapy.cli --update_all
+```
+
 ### Starting the therapy normalization service
 
 From the project root, run the following:
 
-```shell script
- uvicorn main:app --reload
+```commandline
+ uvicorn therapy.main:app --reload
 ```
 
 Next, view the OpenAPI docs on your local machine: 
 
-http://127.0.0.1:8000/docs
+http://127.0.0.1:8000/therapy
diff --git a/setup.cfg b/setup.cfg
index a7216f9b..ab547f58 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -9,7 +9,10 @@ author_email = help@cancervariants.org
 license = MIT
 
 [options]
+packages = find:
 python_requires = >=3.7
+zip_safe = False
+include_package_data = True
 
 install_requires =
     fastapi
@@ -24,5 +27,13 @@ install_requires =
     bs4
     pyyaml
 
+tests_require =
+    pytest
+    pytest-cov
+
+[options.package_data]
+therapy =
+    etl/rxnorm_download.sh
+
 [tool:pytest]
 addopts = --ignore setup.py --doctest-modules --cov-report term-missing --cov .
diff --git a/setup.py b/setup.py
index 5feaec24..48d430d8 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,5 @@
 """Defines how metakb is packaged and distributed."""
-from setuptools import setup, find_packages
+from setuptools import setup
 from therapy import __version__
 
-setup(version=__version__,
-      packages=find_packages(),
-      zip_safe=False)
+setup(version=__version__)
diff --git a/therapy/__init__.py b/therapy/__init__.py
index 76e1decc..60db0292 100644
--- a/therapy/__init__.py
+++ b/therapy/__init__.py
@@ -9,7 +9,7 @@
 logger = logging.getLogger('therapy')
 logger.setLevel(logging.DEBUG)
 
-__version__ = "0.2.1"
+__version__ = "0.2.4"
 
 
 class DownloadException(Exception):
diff --git a/therapy/cli.py b/therapy/cli.py
index 0ab9a7cd..f9a696f6 100644
--- a/therapy/cli.py
+++ b/therapy/cli.py
@@ -2,11 +2,11 @@
 import click
 from botocore.exceptions import ClientError
 from therapy.etl import ChEMBL, Wikidata, DrugBank, NCIt, ChemIDplus, RxNorm
+from therapy.etl.merge import Merge
 from therapy.schemas import SourceName
 from timeit import default_timer as timer
 from therapy.database import Database
 from boto3.dynamodb.conditions import Key
-import sys
 from os import environ
 
 
@@ -19,9 +19,9 @@ class CLI:
         help="The normalizer(s) you wish to update separated by spaces."
     )
     @click.option(
-        '--dev',
+        '--prod',
         is_flag=True,
-        help="Working in development environment on localhost port 8000."
+        help="Working in production environment."
     )
     @click.option(
         '--db_url',
@@ -32,7 +32,14 @@ class CLI:
         is_flag=True,
         help='Update all normalizer sources.'
     )
-    def update_normalizer_db(normalizer, dev, db_url, update_all):
+    @click.option(
+        '--update_merged',
+        is_flag=True,
+        help='Update concepts for normalize endpoint. Must select either '
+             '--update_all or include Mondo as a normalizer source argument.'
+    )
+    def update_normalizer_db(normalizer, prod, db_url, update_all,
+                             update_merged):
         """Update select normalizer source(s) in the therapy database."""
         sources = {
             'chembl': ChEMBL,
@@ -43,25 +50,21 @@ def update_normalizer_db(normalizer, dev, db_url, update_all):
             'rxnorm': RxNorm,
         }
 
-        if dev:
-            db: Database = Database(db_url='http://localhost:8000')
-        elif db_url:
-            db: Database = Database(db_url=db_url)
-        elif 'THERAPY_NORM_DB_URL' in environ.keys():
-            # environment variable will be picked up by DB instance
+        if prod:
+            environ['THERAPY_NORM_PROD'] = "TRUE"
             db: Database = Database()
         else:
-            if click.confirm("Are you sure you want to update"
-                             " the production database?", default=False):
-                click.echo("Updating production db...")
-                db: Database = Database()
+            if db_url:
+                endpoint_url = db_url
+            elif 'THERAPY_NORM_DB_URL' in environ.keys():
+                endpoint_url = environ['THERAPY_NORM_DB_URL']
             else:
-                click.echo("Exiting CLI.")
-                sys.exit()
+                endpoint_url = 'http://localhost:8000'
+            db: Database = Database(db_url=endpoint_url)
 
         if update_all:
             normalizers = list(src for src in sources)
-            CLI()._update_normalizers(normalizers, sources, db)
+            CLI()._update_normalizers(normalizers, sources, db, update_merged)
         elif not normalizer:
             CLI()._help_msg()
         else:
@@ -75,7 +78,7 @@ def update_normalizer_db(normalizer, dev, db_url, update_all):
             if len(non_sources) != 0:
                 raise Exception(f"Not valid source(s): {non_sources}")
 
-            CLI()._update_normalizers(normalizers, sources, db)
+            CLI()._update_normalizers(normalizers, sources, db, update_merged)
 
     def _help_msg(self):
         """Display help message."""
@@ -89,8 +92,9 @@ def _check_norm_srcs_match(self, sources, normalizers):
         """Check that entered normalizers are actual sources."""
         return set(normalizers) - {src for src in sources}
 
-    def _update_normalizers(self, normalizers, sources, db):
+    def _update_normalizers(self, normalizers, sources, db, update_merged):
         """Update selected normalizer sources."""
+        processed_ids = list()
         for n in normalizers:
             click.echo(f"\nDeleting {n}...")
             start_delete = timer()
@@ -102,13 +106,26 @@ def _update_normalizers(self, normalizers, sources, db):
             click.echo(f"Loading {n}...")
             start_load = timer()
             source = sources[n](database=db)
-            source.perform_etl()
+            # TODO: Replace with constant once issue-90 merged
+            if n not in ['chembl', 'drugbank']:
+                processed_ids += source.perform_etl()
+            else:
+                source.perform_etl()
             end_load = timer()
             load_time = end_load - start_load
             click.echo(f"Loaded {n} in {load_time:.5f} seconds.")
             click.echo(f"Total time for {n}: "
                        f"{(delete_time + load_time):.5f} seconds.")
 
+        if update_merged and processed_ids:
+            click.echo("Generating merged concepts...")
+            start_merge = timer()
+            merge = Merge(database=db)
+            merge.create_merged_concepts(processed_ids)
+            end_merge = timer()
+            click.echo(f"Merged concept generation completed in"
+                       f" {(end_merge - start_merge):.5f} seconds.")
+
     def _delete_data(self, source, database):
         # Delete source's metadata
         try:
diff --git a/therapy/database.py b/therapy/database.py
index 9b7b8776..adaa6020 100644
--- a/therapy/database.py
+++ b/therapy/database.py
@@ -6,6 +6,8 @@
 from os import environ
 from typing import Optional, Dict, List, Any
 import logging
+import click
+import sys
 
 logger = logging.getLogger('therapy')
 logger.setLevel(logging.DEBUG)
@@ -26,25 +28,37 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'):
         :param str db_url: database endpoint URL to connect to
         :param str region_name: AWS region name to use
         """
-        if db_url:
+        if 'THERAPY_NORM_PROD' in environ.keys():
             boto_params = {
-                'region_name': region_name,
-                'endpoint_url': db_url
-            }
-        elif 'THERAPY_NORM_DB_URL' in environ.keys():
-            boto_params = {
-                'region_name': region_name,
-                'endpoint_url': environ['THERAPY_NORM_DB_URL']
+                'region_name': region_name
             }
+            if 'THERAPY_NORM_EB_PROD' not in environ.keys():
+                # EB Instance should not have to confirm.
+                # This is used only for using production via CLI
+                if click.confirm("Are you sure you want to use the "
+                                 "production database?", default=False):
+                    click.echo("***PRODUCTION DATABASE IN USE***")
+                else:
+                    click.echo("Exiting.")
+                    sys.exit()
         else:
+            if db_url:
+                endpoint_url = db_url
+            elif 'THERAPY_NORM_DB_URL' in environ.keys():
+                endpoint_url = environ['THERAPY_NORM_DB_URL']
+            else:
+                endpoint_url = 'http://localhost:8000'
+            click.echo(f"***Using Database Endpoint: {endpoint_url}***")
             boto_params = {
-                'region_name': region_name
+                'region_name': region_name,
+                'endpoint_url': endpoint_url
             }
+
         self.dynamodb = boto3.resource('dynamodb', **boto_params)
         self.dynamodb_client = boto3.client('dynamodb', **boto_params)
 
-        # create tables if nonexistant if not connecting to remote database
-        if db_url or 'THERAPY_NORM_DB_URL' in environ.keys():
+        # create tables if nonexistent if not connecting to remote database
+        if 'THERAPY_NORM_PROD' not in environ.keys():
             existing_tables = self.dynamodb_client.list_tables()['TableNames']
             self.create_therapies_table(existing_tables)
             self.create_meta_data_table(existing_tables)
diff --git a/therapy/etl/chemidplus.py b/therapy/etl/chemidplus.py
index 1388a150..fd65062a 100644
--- a/therapy/etl/chemidplus.py
+++ b/therapy/etl/chemidplus.py
@@ -101,10 +101,10 @@ def _extract_data(self):
         self._version = self._data_src.stem.split('_')[1]
 
     def _get_file(self, data_dir):
-        self._download_data()
+        self._download_data(self._data_path)
         dir_files = list(data_dir.iterdir())
         return sorted([f for f in dir_files
-                       if f.names.startswith('chemidplus')])
+                       if f.name.startswith('chemidplus')])
 
     def _transform_data(self):
         """Open dataset and prepare for loading into database."""
diff --git a/therapy/etl/ncit.py b/therapy/etl/ncit.py
index 10eb33b8..63352c87 100644
--- a/therapy/etl/ncit.py
+++ b/therapy/etl/ncit.py
@@ -27,7 +27,7 @@ class NCIt(Base):
 
     def __init__(self,
                  database: Database,
-                 src_dir: str = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/",  # noqa F401
+                 src_dir: str = "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/",  # noqa F401
                  src_fname: str = "Thesaurus_20.09d.OWL.zip",
                  data_path: Path = PROJECT_ROOT / 'data' / 'ncit',
                  chemidplus_path: Path = PROJECT_ROOT / 'data' / 'chemidplus'):
@@ -66,6 +66,7 @@ def _download_data(self):
         for chunk in response.iter_content(chunk_size=512):
             if chunk:
                 handle.write(chunk)
+        handle.close()
         with zipfile.ZipFile(zip_path, 'r') as zip_ref:
             zip_ref.extractall(self._data_path)
         remove(zip_path)
diff --git a/therapy/etl/rxnorm.py b/therapy/etl/rxnorm.py
index 4fbe6c22..2ac81dbd 100644
--- a/therapy/etl/rxnorm.py
+++ b/therapy/etl/rxnorm.py
@@ -6,6 +6,7 @@
  endorse or recommend this or any other product."
 """
 from .base import Base
+import therapy
 from therapy import PROJECT_ROOT, DownloadException
 from therapy.database import Database
 from therapy.schemas import SourceName, NamespacePrefix, Meta, Drug, \
@@ -21,6 +22,7 @@
 import zipfile
 import re
 import yaml
+from pathlib import Path
 
 logger = logging.getLogger('therapy')
 logger.setLevel(logging.DEBUG)
@@ -112,25 +114,27 @@ def _download_data(self, rxn_dir):
             uri = 'https://download.nlm.nih.gov/umls/kss/' \
                   'rxnorm/RxNorm_full_current.zip'
 
+            rxnorm_path = str(Path(therapy.__file__).resolve().parents[0] / 'data' / 'rxnorm' / 'RxNorm_full_current.zip')  # noqa: E501
+            environ['RXNORM_PATH'] = rxnorm_path
+
             # Source:
             # https://documentation.uts.nlm.nih.gov/automating-downloads.html
-            subprocess.call(['bash', f'{PROJECT_ROOT}/therapy/etl/'
+            subprocess.call(['bash', f'{PROJECT_ROOT}/etl/'
                                      f'rxnorm_download.sh', uri])
 
-            zip_path = rxn_dir / 'RxNorm_full_current.zip'
-            shutil.move(PROJECT_ROOT / 'RxNorm_full_current.zip', zip_path)
-
-            with zipfile.ZipFile(zip_path, 'r') as zf:
+            with zipfile.ZipFile(rxnorm_path, 'r') as zf:
                 zf.extractall(rxn_dir)
 
-            remove(zip_path)
+            remove(rxnorm_path)
             shutil.rmtree(rxn_dir / 'prescribe')
             shutil.rmtree(rxn_dir / 'scripts')
 
-            # get version
             readme = sorted([fn for fn in rxn_dir.iterdir() if fn.name.
                             startswith('Readme')])[0]
-            version = str(readme).split('.')[0].split('_')[-1]
+
+            # get version
+            version = str(readme).split('/')[-1].split('.')[0].split('_')[-1]
+
             self._version = datetime.datetime.strptime(
                 version, '%m%d%Y').strftime('%Y%m%d')
             remove(readme)
diff --git a/therapy/etl/rxnorm_download.sh b/therapy/etl/rxnorm_download.sh
index 0c8ab35f..2d87e0f6 100644
--- a/therapy/etl/rxnorm_download.sh
+++ b/therapy/etl/rxnorm_download.sh
@@ -34,5 +34,5 @@ echo $TGTVALUE
 STTICKET=$(curl -d "service="$DOWNLOAD_URL -H "Content-Type: application/x-www-form-urlencoded" -X POST https://utslogin.nlm.nih.gov/cas/v1/tickets/$TGTVALUE)
 echo $STTICKET
 
-curl -c cookie.txt -b cookie.txt -L -O -J $DOWNLOAD_URL?ticket=$STTICKET
+curl -c cookie.txt -b cookie.txt -L -o $RXNORM_PATH -J $DOWNLOAD_URL?ticket=$STTICKET
 rm cookie.txt
\ No newline at end of file
diff --git a/therapy/etl/wikidata.py b/therapy/etl/wikidata.py
index 0ebda81e..17898210 100644
--- a/therapy/etl/wikidata.py
+++ b/therapy/etl/wikidata.py
@@ -78,7 +78,7 @@ def _extract_data(self):
         """Extract data from the Wikidata source."""
         self._data_path.mkdir(exist_ok=True, parents=True)
 
-        data = (wbi_core.ItemEngine.execute_sparql_query(
+        data = (wbi_core.FunctionsEngine.execute_sparql_query(
             SPARQL_QUERY))['results']['bindings']
 
         transformed_data = list()


Test patch:
diff --git a/tests/unit/data/metadata.json b/tests/unit/data/metadata.json
index eeee3861..e032f40c 100644
--- a/tests/unit/data/metadata.json
+++ b/tests/unit/data/metadata.json
@@ -28,7 +28,7 @@
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
     "version": "20.09d",
-    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/",
+    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/",
     "rdp_url": "http://reusabledata.org/ncit.html",
     "data_license_attributes": {
       "non_commercial": false,
diff --git a/tests/unit/test_database.py b/tests/unit/test_database.py
index 2a27ce76..c974f991 100644
--- a/tests/unit/test_database.py
+++ b/tests/unit/test_database.py
@@ -11,7 +11,7 @@ def db():
     """Create a DynamoDB test fixture."""
     class DB:
         def __init__(self):
-            self.db = Database()
+            self.db = Database(db_url=os.environ['THERAPY_NORM_DB_URL'])
             if os.environ.get('TEST') is not None:
                 self.load_test_data()
 
@@ -32,11 +32,11 @@ def load_test_data(self):
                         batch.put_item(Item=m)
                 f.close()
 
-    return DB()
+    return DB().db
 
 
 def test_tables_created(db):
     """Check that therapy_concepts and therapy_metadata are created."""
-    existing_tables = db.db.dynamodb_client.list_tables()['TableNames']
+    existing_tables = db.dynamodb_client.list_tables()['TableNames']
     assert 'therapy_concepts' in existing_tables
     assert 'therapy_metadata' in existing_tables
diff --git a/tests/unit/test_ncit_source.py b/tests/unit/test_ncit_source.py
index 415f2375..0e06fddb 100644
--- a/tests/unit/test_ncit_source.py
+++ b/tests/unit/test_ncit_source.py
@@ -392,7 +392,7 @@ def test_meta_info(voglibose, ncit):
         'https://creativecommons.org/licenses/by/4.0/legalcode'
     assert normalizer_response['meta_'].version == '20.09d'
     assert normalizer_response['meta_'].data_url == \
-        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/"  # noqa: E501
+        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/2020/20.09d_Release/"  # noqa: E501
     assert normalizer_response['meta_'].rdp_url == 'http://reusabledata.org/ncit.html'  # noqa: E501
     assert normalizer_response['meta_'].data_license_attributes == {
         "non_commercial": False,