Repository: cancervariants/metakb (language: python)

Issue:
CIViC harvest fails
```
[ temp-query-deprecation-fix ⚙ .venv] ~/code/metakb % metakb update -r civic                                                                                                                                                                                                                        
Harvesting sources...                                                                                                                                                                                                                                                                                
Harvesting CIViC...                                                                                                                                                                                                                                                                                  
(CIViCPy cache is also being updated)                                                                                                                                                                                                                                                                
Traceback (most recent call last):                                                                                                                                                                                                                                                                   
  File "/Users/jss009/code/metakb/.venv/bin/metakb", line 8, in <module>                                                                                                                                                                                                                             
    sys.exit(cli())                                                                                                                                                                                                                                                                                  
             ^^^^^                                                                                                                                                                                                                                                                                   
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 1205, in __call__                                                                                                                                                                                     
    return anyio.run(self._main, main, args, kwargs, **opts)                                                                                                                                                                                                                                         
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                         
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/anyio/_core/_eventloop.py", line 74, in run                                                                                                                                                                                     
    return async_backend.run(func, args, {}, backend_options)                                                                                                                                                                                                                                        
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                        
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 2034, in run                                                                                                                                                                                 
    return runner.run(wrapper())                                                                                                                                                                                                                                                                     
           ^^^^^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                                                     
  File "/Users/jss009/.pyenv/versions/3.12.2/lib/python3.12/asyncio/runners.py", line 118, in run                                                                                                                                                                                                    
    return self._loop.run_until_complete(task)                                                                                                                                                                                                                                                       
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                                       
  File "/Users/jss009/.pyenv/versions/3.12.2/lib/python3.12/asyncio/base_events.py", line 685, in run_until_complete                                                                                                                                                                                 
    return future.result()                                                                                                                                                                                                                                                                           
           ^^^^^^^^^^^^^^^                                                                                                                                                                                                                                                                           
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 2022, in wrapper                                                                                                                                                                             
    return await func(*args)                                                                                                                                                                                                                                                                         
           ^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                                                         
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 1208, in _main                                                                                                                                                                                        
    return await main(*args, **kwargs)                                                                                                                                                                                                                                                               
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                                                                                                                                                                                                               
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 1120, in main                                     
    rv = await self.invoke(ctx)                                                                                                                  
         ^^^^^^^^^^^^^^^^^^^^^^                                                                                                                  
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 1739, in invoke                                   
    return await _process_result(await sub_ctx.command.invoke(sub_ctx))                                                                          
                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                           
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 1485, in invoke                                   
    return await ctx.invoke(self.callback, **ctx.params)                                                                                         
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                                         
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/asyncclick/core.py", line 824, in invoke                                    
    rv = await rv                                                                                                                                
         ^^^^^^^^                                                                                                                                
  File "/Users/jss009/code/metakb/src/metakb/cli.py", line 511, in update                                                                        
    _harvest_sources(sources, refresh_source_caches)                                                                                             
  File "/Users/jss009/code/metakb/src/metakb/cli.py", line 576, in _harvest_sources                
    source = source_class(update_cache=True, update_from_remote=False)                                                                           
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                                           
  File "/Users/jss009/code/metakb/src/metakb/harvesters/civic.py", line 43, in __init__                                       
    civicpy.update_cache(from_remote_cache=update_from_remote)                                                                                   
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/civicpy/civic.py", line 269, in update_cache  
    genes = _get_elements_by_ids('gene', allow_cached=False, get_all=True)                                          
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                                        
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/civicpy/civic.py", line 1203, in _get_elements_by_ids
    response_elements = _request_all(element)                                                                                                    
                        ^^^^^^^^^^^^^^^^^^^^^                                                                                                    
  File "/Users/jss009/code/metakb/.venv/lib/python3.12/site-packages/civicpy/civic.py", line 1300, in _request_all            
    response = resp.json()['data'][pluralize(element)]                                                                                           
               ~~~~~~~~~~~^^^^^^^^                                                                                                               
KeyError: 'data'
```

this appears to be something about the schema that's fixed in `civicpy` 3.1.0, but because our dependency declaration is "civicpy~=3.0.0", we're not able to install it.


Discussion / hints:

Fix patch:
diff --git a/Pipfile b/Pipfile
index 09ce25b8..df3b5eba 100644
--- a/Pipfile
+++ b/Pipfile
@@ -9,7 +9,7 @@ gene-normalizer = {version = "~=0.3.0-dev1", extras = ["etl"]}
 variation-normalizer = "~=0.8.2"
 disease-normalizer = {version = "~=0.4.0.dev3", extras = ["etl"]}
 thera-py = {version = "~=0.5.0.dev3", extras = ["etl"]}
-civicpy = "~=3.0.0"
+civicpy = "~=3.1"
 requests = "*"
 pydantic = "==2.*"
 requests-cache = "*"
diff --git a/pyproject.toml b/pyproject.toml
index 6d2eafbc..f99dfa4d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
     "variation-normalizer~=0.8.2",
     "disease-normalizer[etl]~=0.4.0.dev3",
     "thera-py[etl]~=0.5.0.dev3",
-    "civicpy~=3.0.0",
+    "civicpy~=3.1",
     "requests",
     "pydantic==2.*",
     "requests-cache",


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index f5a09bfd..f6ff68c5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -467,7 +467,7 @@ def civic_mpid12(civic_vid12, braf_v600e_genomic):
             },
             {
                 "name": "CIViC Molecular Profile Score",
-                "value": 1363.5,
+                "value": 1378.5,
                 "type": "Extension",
             },
             {
@@ -1687,7 +1687,7 @@ def moa_aid66_study(
     moa_imatinib,
     moa_chronic_myelogenous_leukemia,
     moa_method,
-    moa_source44,
+    moa_source45,
 ):
     """Create a Variant Therapeutic Response Study test fixture for MOA Assertion 66."""
     return {
@@ -1705,7 +1705,7 @@ def moa_aid66_study(
         "tumorType": moa_chronic_myelogenous_leukemia,
         "qualifiers": {"alleleOrigin": "somatic", "geneContext": moa_abl1},
         "specifiedBy": moa_method,
-        "isReportedIn": [moa_source44],
+        "isReportedIn": [moa_source45],
         "type": "VariantTherapeuticResponseStudy",
     }
 
@@ -2006,10 +2006,10 @@ def civic_methods(civic_method, moa_method, method3):
 
 
 @pytest.fixture(scope="session")
-def moa_source44():
+def moa_source45():
     """Create a test fixture for MOA source 44."""
     return {
-        "id": "moa.source:44",
+        "id": "moa.source:45",
         "extensions": [
             {"type": "Extension", "name": "source_type", "value": "Journal"}
         ],
diff --git a/tests/data/harvesters/civic/civic_molecular_profile_12.json b/tests/data/harvesters/civic/civic_molecular_profile_12.json
index 36829d74..4a0c7e7d 100644
--- a/tests/data/harvesters/civic/civic_molecular_profile_12.json
+++ b/tests/data/harvesters/civic/civic_molecular_profile_12.json
@@ -5,7 +5,7 @@
     12
   ],
   "name": "BRAF V600E",
-  "molecular_profile_score": 1363.5,
+  "molecular_profile_score": 1378.5,
   "description": "BRAF V600E has been shown to be recurrent in many cancer types. It is one of the most widely studied variants in cancer. This variant is correlated with poor prognosis in certain cancer types, including colorectal cancer and papillary thyroid cancer. The targeted therapeutic dabrafenib has been shown to be effective in clinical trials with an array of BRAF mutations and cancer types. Dabrafenib has also shown to be effective when combined with the MEK inhibitor trametinib in colorectal cancer and melanoma. However, in patients with TP53, CDKN2A and KRAS mutations, dabrafenib resistance has been reported. Ipilimumab, regorafenib, vemurafenib, and a number of combination therapies have been successful in treating V600E mutations. However, cetuximab and panitumumab have been largely shown to be ineffective without supplementary treatment.",
   "sources": [],
   "aliases": [
@@ -13,5 +13,18 @@
     "VAL600GLU",
     "V640E",
     "VAL640GLU"
+  ],
+  "parsed_name": [
+    {
+      "id": 5,
+      "name": "BRAF",
+      "type": "feature"
+    },
+    {
+      "deprecated": false,
+      "id": 12,
+      "name": "V600E",
+      "type": "variant"
+    }
   ]
 }
diff --git a/tests/data/harvesters/civic/civicpy_cache_20240325.pkl b/tests/data/harvesters/civic/civicpy_cache_20240612.pkl
similarity index 63%
rename from tests/data/harvesters/civic/civicpy_cache_20240325.pkl
rename to tests/data/harvesters/civic/civicpy_cache_20240612.pkl
index 74487d9d..3d816126 100644
Binary files a/tests/data/harvesters/civic/civicpy_cache_20240325.pkl and b/tests/data/harvesters/civic/civicpy_cache_20240612.pkl differ
diff --git a/tests/data/transform/moa_harvester.json b/tests/data/transform/moa_harvester.json
index 9dbb07f6..562e3c9b 100644
--- a/tests/data/transform/moa_harvester.json
+++ b/tests/data/transform/moa_harvester.json
@@ -18,7 +18,7 @@
 			"last_updated": "2023-11-30",
 			"submitted_by": "breardon@broadinstitute.org",
 			"validated": true,
-			"source_ids": 44,
+			"source_ids": 45,
 			"variant": {
 				"id": 66,
 				"alternate_allele": "T",
@@ -75,7 +75,7 @@
 	],
 	"sources": [
 		{
-			"id": 44,
+			"id": 45,
 			"type": "Journal",
 			"doi": "10.1126/science.1062538",
 			"nct": "",
@@ -131,4 +131,4 @@
 		"ABL1",
 		"BRAF"
 	]
-}
\ No newline at end of file
+}
diff --git a/tests/data/transform/therapeutic/civic_harvester.json b/tests/data/transform/therapeutic/civic_harvester.json
index 1885f37b..a5a694cc 100644
--- a/tests/data/transform/therapeutic/civic_harvester.json
+++ b/tests/data/transform/therapeutic/civic_harvester.json
@@ -566,7 +566,7 @@
         12
       ],
       "name": "BRAF V600E",
-      "molecular_profile_score": 1363.5,
+      "molecular_profile_score": 1378.5,
       "description": "BRAF V600E has been shown to be recurrent in many cancer types. It is one of the most widely studied variants in cancer. This variant is correlated with poor prognosis in certain cancer types, including colorectal cancer and papillary thyroid cancer. The targeted therapeutic dabrafenib has been shown to be effective in clinical trials with an array of BRAF mutations and cancer types. Dabrafenib has also shown to be effective when combined with the MEK inhibitor trametinib in colorectal cancer and melanoma. However, in patients with TP53, CDKN2A and KRAS mutations, dabrafenib resistance has been reported. Ipilimumab, regorafenib, vemurafenib, and a number of combination therapies have been successful in treating V600E mutations. However, cetuximab and panitumumab have been largely shown to be ineffective without supplementary treatment.",
       "sources": [],
       "aliases": [
@@ -577,4 +577,4 @@
       ]
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/tests/unit/database/test_database.py b/tests/unit/database/test_database.py
index b88126b0..9fb934b2 100644
--- a/tests/unit/database/test_database.py
+++ b/tests/unit/database/test_database.py
@@ -555,7 +555,7 @@ def test_document_rules(
     check_node_labels,
     check_relation_count,
     get_node_by_id,
-    moa_source44,
+    moa_source45,
     check_node_props,
     check_extension_props,
 ):
@@ -589,11 +589,11 @@ def test_document_rules(
         record = s.run(is_reported_in_query).single()
     assert set(record.values()[0]) == {31779674, 35121878}
 
-    doc = get_node_by_id(moa_source44["id"])
+    doc = get_node_by_id(moa_source45["id"])
     extension_names = {"source_type"}
-    check_extension_props(doc, moa_source44["extensions"], extension_names)
+    check_extension_props(doc, moa_source45["extensions"], extension_names)
     expected_keys = {"id", "title", "doi", "source_type", "url", "pmid"}
-    check_node_props(doc, moa_source44, expected_keys, extension_names)
+    check_node_props(doc, moa_source45, expected_keys, extension_names)
 
 
 def test_method_rules(
diff --git a/tests/unit/test_search_studies.py b/tests/unit/test_search_studies.py
index 22834f7b..58fce8f7 100644
--- a/tests/unit/test_search_studies.py
+++ b/tests/unit/test_search_studies.py
@@ -4,7 +4,6 @@
 
 from metakb.query import QueryHandler
 from metakb.schemas.api import SearchStudiesService
-from metakb.schemas.app import SourceName
 
 
 @pytest.fixture(scope="module")
@@ -74,9 +73,6 @@ async def test_civic_eid2997(query_handler, civic_eid2997_study, assertion_check
     # genomic query
     resp = await query_handler.search_studies(variation="7-55259515-T-G")
     find_and_check_study(resp, civic_eid2997_study, assertion_checks)
-    # At the moment, MOA cannot be queried via related genomic queries
-    # only civic stores genomic members
-    assert not [s_id for s_id in resp.study_ids if s_id.startswith(SourceName.MOA)]
 
     resp = await query_handler.search_studies(therapy="ncit:C66940")
     find_and_check_study(resp, civic_eid2997_study, assertion_checks)