Repository: cancervariants/therapy-normalization (language: python)

Issue:
Update source meta information
Add additional attributes to source meta:

-     rdp_url: Optional[str]
-     non_commercial: Optional[bool]
-     share_alike: Optional[bool]
-     attribution: Optional[bool]

Create data_license_attributes field
We should bundle `share_alike`, `non_commercial`, `attribution` under the `data_license_attributes` field. 


Discussion / hints:

Fix patch:
diff --git a/dynamodb_revisions/update_meta.py b/dynamodb_revisions/update_meta.py
new file mode 100644
index 00000000..5894783f
--- /dev/null
+++ b/dynamodb_revisions/update_meta.py
@@ -0,0 +1,68 @@
+"""Update source metadata in prod"""
+import sys
+from pathlib import Path
+import click
+from os import environ
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+sys.path.append(f"{PROJECT_ROOT}")
+
+from therapy.database import Database  # noqa: E402
+from therapy.schemas import SourceName  # noqa: E402
+
+
+def main():
+    """Update DB with new source meta info."""
+    db = Database()
+    updates = {
+        SourceName.CHEMBL.value: {
+            "rdp_url": "http://reusabledata.org/chembl.html",
+            "data_license_attributes": {
+                "non_commercial": False,
+                "share_alike": True,
+                "attribution": True
+            }
+        },
+        SourceName.DRUGBANK.value: {
+            "rdp_url": "http://reusabledata.org/drugbank.html",
+            "data_license_attributes": {
+                "non_commercial": True,
+                "share_alike": False,
+                "attribution": True
+            }
+        },
+        SourceName.NCIT.value: {
+            "rdp_url": "http://reusabledata.org/ncit.html",
+            "data_license_attributes": {
+                "non_commercial": False,
+                "share_alike": False,
+                "attribution": True
+            }
+        },
+        SourceName.WIKIDATA.value: {
+            "rdp_url": None,
+            "data_license_attributes": {
+                "non_commercial": False,
+                "share_alike": False,
+                "attribution": False
+            }
+        }
+    }
+
+    for src_name in updates.keys():
+        attrs = {k: {'Value': v} for k, v in updates[src_name].items()}
+        db.metadata.update_item(
+            Key={'src_name': src_name},
+            AttributeUpdates=attrs
+        )
+
+
+if __name__ == '__main__':
+    if 'THERAPY_NORM_DB_URL' not in environ.keys():
+        if click.confirm("Are you sure you want to update"
+                         " the production database?", default=False):
+            click.echo("Updating production db...")
+        else:
+            click.echo("Exiting.")
+            sys.exit()
+    main()
diff --git a/therapy/etl/chembl.py b/therapy/etl/chembl.py
index e3f19646..de60b205 100644
--- a/therapy/etl/chembl.py
+++ b/therapy/etl/chembl.py
@@ -4,7 +4,7 @@
 from ftplib import FTP
 import logging
 import tarfile  # noqa: F401
-from therapy.schemas import SourceName, NamespacePrefix, ApprovalStatus
+from therapy.schemas import SourceName, NamespacePrefix, ApprovalStatus, Meta
 import sqlite3
 
 logger = logging.getLogger('therapy')
@@ -263,14 +263,16 @@ def _load_trade_name(self, record, batch):
 
     def _add_meta(self, *args, **kwargs):
         """Add ChEMBL metadata."""
-        self.database.metadata.put_item(
-            Item={
-                'src_name': SourceName.CHEMBL.value,
-                'data_license': 'CC BY-SA 3.0',
-                'data_license_url':
-                    'https://creativecommons.org/licenses/by-sa/3.0/',
-                'version': '27',
-                'data_url':
-                    'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/'  # noqa: E501
-            }
-        )
+        metadata = Meta(data_license='CC BY-SA 3.0',
+                        data_license_url='https://creativecommons.org/licenses/by-sa/3.0/',  # noqa: E501
+                        version='27',
+                        data_url='http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/',  # noqa: E501
+                        rdp_url='http://reusabledata.org/chembl.html',
+                        data_license_attributes={
+                            'non_commercial': False,
+                            'share_alike': True,
+                            'attribution': True
+                        })
+        params = dict(metadata)
+        params['src_name'] = SourceName.CHEMBL.value
+        self.database.metadata.put_item(Item=params)
diff --git a/therapy/etl/drugbank.py b/therapy/etl/drugbank.py
index c588f4dc..92b1817b 100644
--- a/therapy/etl/drugbank.py
+++ b/therapy/etl/drugbank.py
@@ -2,7 +2,7 @@
 from therapy.etl.base import Base
 from therapy import PROJECT_ROOT
 import logging
-from therapy.schemas import SourceName, NamespacePrefix, ApprovalStatus
+from therapy.schemas import SourceName, NamespacePrefix, ApprovalStatus, Meta
 from therapy.etl.base import IDENTIFIER_PREFIXES
 from lxml import etree
 
@@ -247,14 +247,16 @@ def _load_trade_names(self, trade_names, concept_id, batch):
 
     def _add_meta(self):
         """Add DrugBank metadata."""
-        self.database.metadata.put_item(
-            Item={
-                'src_name': SourceName.DRUGBANK.value,
-                'data_license': 'CC BY-NC 4.0',
-                'data_license_url':
-                    'https://creativecommons.org/licenses/by-nc/4.0/legalcode',
-                'version': '5.1.7',
-                'data_url':
-                    'https://go.drugbank.com/releases/5-1-7/downloads/all-full-database'  # noqa E501
-            }
-        )
+        meta = Meta(data_license='CC BY-NC 4.0',
+                    data_license_url='https://creativecommons.org/licenses/by-nc/4.0/legalcode',  # noqa: E501
+                    version='5.1.7',
+                    data_url='https://go.drugbank.com/releases/5-1-7/downloads/all-full-database',  # noqa: E501
+                    rdp_url='http://reusabledata.org/drugbank.html',
+                    data_license_attributes={
+                        'non_commercial': True,
+                        'share_alike': False,
+                        'attribution': True
+                    })
+        params = dict(meta)
+        params['src_name'] = SourceName.DRUGBANK.value
+        self.database.metadata.put_item(Item=params)
diff --git a/therapy/etl/ncit.py b/therapy/etl/ncit.py
index 4153b429..01419626 100644
--- a/therapy/etl/ncit.py
+++ b/therapy/etl/ncit.py
@@ -183,14 +183,16 @@ def _add_meta(self):
         metadata = Meta(data_license="CC BY 4.0",
                         data_license_url="https://creativecommons.org/licenses/by/4.0/legalcode",  # noqa F401
                         version=self._version,
-                        data_url=self._SRC_DIR)
-        self.database.metadata.put_item(Item={
-            'src_name': SourceName.NCIT.value,
-            'data_license': metadata.data_license,
-            'data_license_url': metadata.data_license_url,
-            'version': metadata.version,
-            'data_url': metadata.data_url
-        })
+                        data_url=self._SRC_DIR,
+                        rdp_url='http://reusabledata.org/ncit.html',
+                        data_license_attributes={
+                            'non_commercial': False,
+                            'share_alike': False,
+                            'attribution': True
+                        })
+        params = dict(metadata)
+        params['src_name'] = SourceName.NCIT.value
+        self.database.metadata.put_item(Item=params)
 
     def _load_therapy(self, therapy: Therapy, batch):
         """Load individual therapy into dynamodb table
diff --git a/therapy/etl/wikidata.py b/therapy/etl/wikidata.py
index 0f736078..129e0427 100644
--- a/therapy/etl/wikidata.py
+++ b/therapy/etl/wikidata.py
@@ -73,14 +73,16 @@ def _add_meta(self):
                         data_license='CC0 1.0',
                         data_license_url='https://creativecommons.org/publicdomain/zero/1.0/',  # noqa: E501
                         version=self._version,
-                        data_url=None)
-        self.database.metadata.put_item(Item={
-            'src_name': SourceName.WIKIDATA.value,
-            'data_license': metadata.data_license,
-            'data_license_url': metadata.data_license_url,
-            'version': metadata.version,
-            'data_url': metadata.data_url
-        })
+                        data_url=None,
+                        rdp_url=None,
+                        data_license_attributes={
+                            'non_commercial': False,
+                            'share_alike': False,
+                            'attribution': False
+                        })
+        params = dict(metadata)
+        params['src_name'] = SourceName.WIKIDATA.value
+        self.database.metadata.put_item(Item=params)
 
     def _transform_data(self):
         """Transform the Wikidata source data.
diff --git a/therapy/schemas.py b/therapy/schemas.py
index 097c1091..11c62347 100644
--- a/therapy/schemas.py
+++ b/therapy/schemas.py
@@ -2,7 +2,7 @@
 therapy records.
 """
 from typing import List, Optional, Dict, Union, Any, Type
-from pydantic import BaseModel
+from pydantic import BaseModel, StrictBool
 from enum import Enum, IntEnum
 
 
@@ -154,6 +154,14 @@ class NamespacePrefix(Enum):
     GUIDETOPHARMACOLOGY = "gtopdb"
 
 
+class DataLicenseAttributes(BaseModel):
+    """Define constraints for data license attributes."""
+
+    non_commercial: StrictBool
+    share_alike: StrictBool
+    attribution: StrictBool
+
+
 class Meta(BaseModel):
     """Metadata for a given source to return in response object."""
 
@@ -161,6 +169,8 @@ class Meta(BaseModel):
     data_license_url: str
     version: str
     data_url: Optional[str]
+    rdp_url: Optional[str]
+    data_license_attributes: Dict[str, StrictBool]
 
     class Config:
         """Enables orm_mode"""
@@ -179,7 +189,13 @@ def schema_extra(schema: Dict[str, Any],
                     'https://creativecommons.org/licenses/by-sa/3.0/',
                 'version': '27',
                 'data_url':
-                    'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/'  # noqa: E501
+                    'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/',  # noqa: E501
+                'rdp_url': 'http://reusabledata.org/chembl.html',
+                'data_license_attributes': {
+                    'non_commercial': False,
+                    'share_alike': True,
+                    'attribution': True
+                }
             }
 
 
@@ -213,6 +229,12 @@ def schema_extra(schema: Dict[str, Any],
                     'version': '27',
                     'data_url':
                         'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/',  # noqa: E501
+                    'rdp_url': 'http://reusabledata.org/chembl.html',
+                    'data_license_attributes': {
+                        'non_commercial': False,
+                        'share_alike': True,
+                        'attribution': True
+                    }
                 },
             }
 
@@ -249,6 +271,12 @@ def schema_extra(schema: Dict[str, Any],
                     'version': '27',
                     'data_url':
                         'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/',  # noqa: E501
+                    'rdp_url': 'http://reusabledata.org/chembl.html',
+                    'data_license_attributes': {
+                        'non_commercial': False,
+                        'share_alike': True,
+                        'attribution': True
+                    }
                 },
             }
 
@@ -280,6 +308,12 @@ def schema_extra(schema: Dict[str, Any],
                         'https://creativecommons.org/licenses/by-sa/3.0/',
                     'version': '27',
                     'data_url':
-                        'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/'  # noqa: E501
+                        'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/',  # noqa: E501
+                    'rdp_url': 'http://reusabledata.org/chembl.html',
+                    'data_license_attributes': {
+                        'non_commercial': False,
+                        'share_alike': True,
+                        'attribution': True
+                    }
                 }
             }


Test patch:
diff --git a/tests/unit/data/metadata.json b/tests/unit/data/metadata.json
index 92b8d2f5..45b75f60 100644
--- a/tests/unit/data/metadata.json
+++ b/tests/unit/data/metadata.json
@@ -4,27 +4,51 @@
     "data_license": "CC BY-SA 3.0",
     "data_license_url": "https://creativecommons.org/licenses/by-sa/3.0/",
     "version": "27",
-    "data_url": "http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/"
+    "data_url": "http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/",
+    "rdp_url": "http://reusabledata.org/chembl.html",
+    "data_license_attributes": {
+      "non_commercial": false,
+      "share_alike": true,
+      "attribution": true
+    }
   },
   {
     "src_name": "Wikidata",
     "data_license": "CC0 1.0",
     "data_license_url": "https://creativecommons.org/publicdomain/zero/1.0/",
     "version": "20200812",
-    "data_url": null
+    "data_url": null,
+    "rdp_url": null,
+    "data_license_attributes": {
+      "non_commercial": false,
+      "share_alike": false,
+      "attribution": false
+    }
   },
   {
     "src_name": "NCIt",
     "data_license": "CC BY 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by/4.0/legalcode",
     "version": "20.09d",
-    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/"
+    "data_url": "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/",
+    "rdp_url": "http://reusabledata.org/ncit.html",
+    "data_license_attributes": {
+      "non_commercial": false,
+      "share_alike": false,
+      "attribution": true
+    }
   },
   {
     "src_name": "DrugBank",
     "data_license": "CC BY-NC 4.0",
     "data_license_url": "https://creativecommons.org/licenses/by-nc/4.0/legalcode",
     "version": "5.1.7",
-    "data_url": "https://go.drugbank.com/releases/5-1-7/downloads/all-full-database"
+    "data_url": "https://go.drugbank.com/releases/5-1-7/downloads/all-full-database",
+    "rdp_url": "http://reusabledata.org/drugbank.html",
+    "data_license_attributes": {
+      "non_commercial": true,
+      "share_alike": false,
+      "attribution": true
+    }
   }
-]
\ No newline at end of file
+]
diff --git a/tests/unit/test_chembl_source.py b/tests/unit/test_chembl_source.py
index d45a40f7..b89bcaf8 100644
--- a/tests/unit/test_chembl_source.py
+++ b/tests/unit/test_chembl_source.py
@@ -461,3 +461,9 @@ def test_meta_info(cisplatin, chembl):
     assert normalizer_response['meta_'].version == '27'
     assert normalizer_response['meta_'].data_url == \
            'http://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/chembl_27/'  # noqa: E501
+    assert normalizer_response['meta_'].rdp_url == 'http://reusabledata.org/chembl.html'  # noqa: E501
+    assert normalizer_response['meta_'].data_license_attributes == {
+        "non_commercial": False,
+        "share_alike": True,
+        "attribution": True
+    }
diff --git a/tests/unit/test_drugbank_source.py b/tests/unit/test_drugbank_source.py
index f2c1a66c..727e6723 100644
--- a/tests/unit/test_drugbank_source.py
+++ b/tests/unit/test_drugbank_source.py
@@ -626,3 +626,9 @@ def test_meta_info(cisplatin, drugbank):
     assert normalizer_response.version == '5.1.7'
     assert normalizer_response.data_url == \
            'https://go.drugbank.com/releases/5-1-7/downloads/all-full-database'  # noqa: E501
+    assert normalizer_response.rdp_url == 'http://reusabledata.org/drugbank.html'  # noqa: E501
+    assert normalizer_response.data_license_attributes == {
+        "non_commercial": True,
+        "share_alike": False,
+        "attribution": True
+    }
diff --git a/tests/unit/test_ncit_source.py b/tests/unit/test_ncit_source.py
index b71f953d..fb6c087f 100644
--- a/tests/unit/test_ncit_source.py
+++ b/tests/unit/test_ncit_source.py
@@ -392,4 +392,10 @@ def test_meta_info(voglibose, ncit):
         'https://creativecommons.org/licenses/by/4.0/legalcode'
     assert normalizer_response['meta_'].version == '20.09d'
     assert normalizer_response['meta_'].data_url == \
-        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/"  # noqa F401
+        "https://evs.nci.nih.gov/ftp1/NCI_Thesaurus/archive/20.09d_Release/"  # noqa: E501
+    assert normalizer_response['meta_'].rdp_url == 'http://reusabledata.org/ncit.html'  # noqa: E501
+    assert normalizer_response['meta_'].data_license_attributes == {
+        "non_commercial": False,
+        "share_alike": False,
+        "attribution": True
+    }
diff --git a/tests/unit/test_wikidata_source.py b/tests/unit/test_wikidata_source.py
index b9022de3..f7d0cde6 100644
--- a/tests/unit/test_wikidata_source.py
+++ b/tests/unit/test_wikidata_source.py
@@ -610,3 +610,9 @@ def test_meta_info(cisplatin, wikidata):
            'https://creativecommons.org/publicdomain/zero/1.0/'
     assert normalizer_response['meta_'].version == '20200812'
     assert normalizer_response['meta_'].data_url is None
+    assert not normalizer_response['meta_'].rdp_url
+    assert normalizer_response['meta_'].data_license_attributes == {
+        "non_commercial": False,
+        "share_alike": False,
+        "attribution": False
+    }