Repository: mlflow/mlflow

Issue:
[MLflow 1.1] Loading models with mlflow.pyfunc.spark_udf broken on Databricks clusters
### System information
- **Have I written custom code (as opposed to using a stock example script provided in MLflow)**: No
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 16.04
- **MLflow installed from (source or binary)**:  Binary
- **MLflow version (run ``mlflow --version``)**: 1.1.0
- **Python version**: 3.6
- **npm version (if running the dev UI): N/A
- **Exact command to reproduce**: Try to load a model as a Spark UDF via `mlflow.pyfunc.spark_udf`

### Describe the problem
Loading models using a Spark UDF is broken in MLflow 1.1 due to a bug in zipping files from DBFS FUSE (the timestamps on the files are misconfigured or read incorrectly).

As a workaround, you can disable the DBFS FUSE logic before loading your model via:
```
import os
os.environ["MLFLOW_ENABLE_DBFS_FUSE_ARTIFACT_REPO"] = 'false'
```


### Source code / logs
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<command-3584523> in <module>()
     1 import mlflow.pyfunc
     2
----> 3 trained_model = mlflow.pyfunc.spark_udf(spark, model_uri='runs:/c4569266b782467cacd91c3d0ef8f9fc/model')

/databricks/python/lib/python3.6/site-packages/mlflow/pyfunc/__init__.py in spark_udf(spark, model_uri, result_type)
   416
   417     local_model_path = _download_artifact_from_uri(artifact_uri=model_uri)
--> 418     archive_path = SparkModelCache.add_local_model(spark, local_model_path)
   419
   420     def predict(*args):

/databricks/python/lib/python3.6/site-packages/mlflow/pyfunc/spark_model_cache.py in add_local_model(spark, model_path)
    35         # NB: We must archive the directory as Spark.addFile does not support non-DFS
    36         # directories when recursive=True.
---> 37         archive_path = shutil.make_archive(archive_basepath, 'zip', model_path)
    38         spark.sparkContext.addFile(archive_path)
    39         return archive_path

/databricks/python/lib/python3.6/shutil.py in make_archive(base_name, format, root_dir, base_dir, verbose, dry_run, owner, group, logger)
   798
   799     try:
--> 800         filename = func(base_name, base_dir, **kwargs)
   801     finally:
   802         if root_dir is not None:

/databricks/python/lib/python3.6/shutil.py in _make_zipfile(base_name, base_dir, verbose, dry_run, logger)
   690                 for name in sorted(dirnames):
   691                     path = os.path.normpath(os.path.join(dirpath, name))
--> 692                     zf.write(path, path)
   693                     if logger is not None:
   694                         logger.info("adding '%s'", path)

/databricks/python/lib/python3.6/zipfile.py in write(self, filename, arcname, compress_type)
  1592             )
  1593
-> 1594         zinfo = ZipInfo.from_file(filename, arcname)
  1595
  1596         if zinfo.is_dir():

/databricks/python/lib/python3.6/zipfile.py in from_file(cls, filename, arcname)
   494         if isdir:
   495             arcname += '/'
--> 496         zinfo = cls(arcname, date_time)
   497         zinfo.external_attr = (st.st_mode & 0xFFFF) << 16  # Unix attributes
   498         if isdir:

/databricks/python/lib/python3.6/zipfile.py in __init__(self, filename, date_time)
   336
   337         if date_time[0] < 1980:
--> 338             raise ValueError('ZIP does not support timestamps before 1980')
   339
   340         # Standard values:

ValueError: ZIP does not support timestamps before 1980
```


Discussion:

Resolving patch:
diff --git a/mlflow/pyfunc/__init__.py b/mlflow/pyfunc/__init__.py
--- a/mlflow/pyfunc/__init__.py
+++ b/mlflow/pyfunc/__init__.py
@@ -414,8 +414,10 @@ def spark_udf(spark, model_uri, result_type="double"):
                     "of the following types types: {}".format(str(elem_type), str(supported_types)),
             error_code=INVALID_PARAMETER_VALUE)
 
-    local_model_path = _download_artifact_from_uri(artifact_uri=model_uri)
-    archive_path = SparkModelCache.add_local_model(spark, local_model_path)
+    with TempDir() as local_tmpdir:
+        local_model_path = _download_artifact_from_uri(
+            artifact_uri=model_uri, output_path=local_tmpdir.path())
+        archive_path = SparkModelCache.add_local_model(spark, local_model_path)
 
     def predict(*args):
         model = SparkModelCache.get_or_load(archive_path)


Test patch:
diff --git a/tests/projects/test_projects.py b/tests/projects/test_projects.py
--- a/tests/projects/test_projects.py
+++ b/tests/projects/test_projects.py
@@ -1,5 +1,6 @@
 import os
 import git
+import shutil
 import tempfile
 import yaml
 
@@ -43,6 +44,14 @@ def _get_version_local_git_repo(local_git_repo):
     return repo.git.rev_parse("HEAD")
 
 
+@pytest.fixture(scope="module", autouse=True)
+def clean_mlruns_dir():
+    yield
+    dir_path = os.path.join(TEST_PROJECT_DIR, "mlruns")
+    if os.path.exists(dir_path):
+        shutil.rmtree(dir_path)
+
+
 @pytest.fixture
 def local_git_repo(tmpdir):
     local_git = tmpdir.join('git_repo').strpath
diff --git a/tests/projects/test_projects_cli.py b/tests/projects/test_projects_cli.py
--- a/tests/projects/test_projects_cli.py
+++ b/tests/projects/test_projects_cli.py
@@ -2,6 +2,7 @@
 import hashlib
 import mock
 import os
+import shutil
 import logging
 
 from click.testing import CliRunner
@@ -26,6 +27,14 @@ def test_run_local_params(tracking_uri_mock):  # pylint: disable=unused-argument
                                 "-P", "excitement=%s" % excitement_arg])
 
 
+@pytest.fixture(scope="module", autouse=True)
+def clean_mlruns_dir():
+    yield
+    dir_path = os.path.join(TEST_PROJECT_DIR, "mlruns")
+    if os.path.exists(dir_path):
+        shutil.rmtree(dir_path)
+
+
 @pytest.mark.large
 def test_run_local_conda_env(tracking_uri_mock):  # pylint: disable=unused-argument
     with open(os.path.join(TEST_PROJECT_DIR, "conda.yaml"), "r") as handle: