refacor loading

configuration.py

@@ -1,11 +1,11 @@
 """Configuration utils."""
 import os
 import json
-import dill
 import pandas as pd
 from pytoda.transforms import Compose
 from pytoda.smiles.transforms import SMILESToTokenIndexes, LeftPadding, Canonicalization
 from cos import ensure_filepath_from_uri, COS_BUCKET_URI
+from .utils import load
 
 # model files
 MODEL_WEIGHTS_URI = ensure_filepath_from_uri(os.path.join(COS_BUCKET_URI, "model.pt"))
@@ -29,16 +29,13 @@ with open(MODEL_PARAMS_URI) as fp:
     MODEL_PARAMS = json.load(fp)
 MAX_LENGTH = MODEL_PARAMS["smiles_padding_length"]
 # load SMILES language
-with open(SMILES_LANGUAGE_URI, "rb") as fp:
-    SMILES_LANGUAGE = dill.load(fp)
+SMILES_LANGUAGE = load(SMILES_LANGUAGE_URI)
 # load gene expression
 GENE_EXPRESSION = pd.read_csv(GENE_EXPRESSION_URI, compression="zip", low_memory=False)
 # load genes
-with open(GENES_URI, "rb") as fp:
-    GENES = dill.load(fp)
+GENES = load(GENES_URI)
 # load gene standardization parameters
-with open(GENE_EXPRESSION_STANDARDIZATION_URI, "rb") as fp:
-    GENE_STANDARDIZATION_PARAMETERS = dill.load(fp)
+GENE_STANDARDIZATION_PARAMETERS = load(GENE_EXPRESSION_STANDARDIZATION_URI)
 # smiles transformations
 SMILES_TRANSFORMS = [
     Canonicalization(),

utils.py

@@ -10,6 +10,31 @@ from datetime import datetime  # noqa: F401
 from typing import Dict, List  # noqa: F401
 
 import six
+import warnings
+import dill
+
+def load(filepath: str):
+    """
+    Load a pickled object with Python downwards compatibility
+
+    Args:
+        filepath (str): path to the file.
+    """
+    warnings.warn(
+        "Loading languages will use a text files in the future", FutureWarning
+    )
+    try:
+        with open(filepath, 'rb') as f:
+            obj = dill.load(f)
+    except TypeError:
+        # Necessary to load python3.7 pickled objects with >=3.8:
+        # For details see: https://github.com/uqfoundation/dill/pull/406
+        storage = dill._dill._reverse_typemap['CodeType']
+        dill._dill._reverse_typemap['CodeType'] = dill._dill._create_code
+        with open(filepath, 'rb') as f:
+            obj = dill.load(f)
+        dill._dill._reverse_typemap['CodeType'] = storage
+    return obj
 
 if sys.version_info < (3, 7):
     import typing