Repository: databricks/koalas

Issue:
Re-enable PyArrow 0.15 support back
We should enable https://github.com/databricks/koalas/pull/1108 back once we have enough documentation and error messages.


Discussion:

Resolving patch:
diff --git a/databricks/koalas/__init__.py b/databricks/koalas/__init__.py
--- a/databricks/koalas/__init__.py
+++ b/databricks/koalas/__init__.py
@@ -45,12 +45,30 @@ def assert_pyspark_version():
 import pyspark
 import pyarrow
 
-if LooseVersion(pyarrow.__version__) >= LooseVersion("0.15") and LooseVersion(
-    pyspark.__version__
-) < LooseVersion("3.0"):
-    # This is required to support PyArrow 0.15 in PySpark versions lower than 3.0.
-    # See SPARK-29367.
-    os.environ["ARROW_PRE_0_15_IPC_FORMAT"] = "1"
+if LooseVersion(pyspark.__version__) < LooseVersion("3.0"):
+    if (
+        LooseVersion(pyarrow.__version__) >= LooseVersion("0.15")
+        and "ARROW_PRE_0_15_IPC_FORMAT" not in os.environ
+    ):
+        import logging
+
+        logging.warning(
+            "'ARROW_PRE_0_15_IPC_FORMAT' environment variable was not set. It is required to "
+            "set this environment variable to '1' in both driver and executor sides if you use "
+            "pyarrow>=0.15 and pyspark<3.0. "
+            "Koalas will set it for you but it does not work if there is a Spark context already "
+            "launched."
+        )
+        # This is required to support PyArrow 0.15 in PySpark versions lower than 3.0.
+        # See SPARK-29367.
+        os.environ["ARROW_PRE_0_15_IPC_FORMAT"] = "1"
+elif "ARROW_PRE_0_15_IPC_FORMAT" in os.environ:
+    raise RuntimeError(
+        "Please explicitly unset 'ARROW_PRE_0_15_IPC_FORMAT' environment variable in both "
+        "driver and executor sides. It is required to set this environment variable only "
+        "when you use pyarrow>=0.15 and pyspark<3.0."
+    )
+
 
 from databricks.koalas.frame import DataFrame
 from databricks.koalas.indexes import Index, MultiIndex
diff --git a/databricks/koalas/utils.py b/databricks/koalas/utils.py
--- a/databricks/koalas/utils.py
+++ b/databricks/koalas/utils.py
@@ -296,6 +296,7 @@ def align_diff_series(func, this_series, *args, how="full"):
 def default_session(conf=None):
     if conf is None:
         conf = dict()
+    should_use_legacy_ipc = False
     if LooseVersion(pyarrow.__version__) >= LooseVersion("0.15") and LooseVersion(
         pyspark.__version__
     ) < LooseVersion("3.0"):
@@ -303,13 +304,35 @@ def default_session(conf=None):
         conf["spark.yarn.appMasterEnv.ARROW_PRE_0_15_IPC_FORMAT"] = "1"
         conf["spark.mesos.driverEnv.ARROW_PRE_0_15_IPC_FORMAT"] = "1"
         conf["spark.kubernetes.driverEnv.ARROW_PRE_0_15_IPC_FORMAT"] = "1"
+        should_use_legacy_ipc = True
+
     builder = spark.SparkSession.builder.appName("Koalas")
     for key, value in conf.items():
         builder = builder.config(key, value)
     # Currently, Koalas is dependent on such join due to 'compute.ops_on_diff_frames'
     # configuration. This is needed with Spark 3.0+.
     builder.config("spark.sql.analyzer.failAmbiguousSelfJoin.enabled", False)
-    return builder.getOrCreate()
+    session = builder.getOrCreate()
+
+    if not should_use_legacy_ipc:
+        is_legacy_ipc_set = any(
+            v == "1"
+            for v in [
+                session.conf.get("spark.executorEnv.ARROW_PRE_0_15_IPC_FORMAT", None),
+                session.conf.get("spark.yarn.appMasterEnv.ARROW_PRE_0_15_IPC_FORMAT", None),
+                session.conf.get("spark.mesos.driverEnv.ARROW_PRE_0_15_IPC_FORMAT", None),
+                session.conf.get("spark.kubernetes.driverEnv.ARROW_PRE_0_15_IPC_FORMAT", None),
+            ]
+        )
+        if is_legacy_ipc_set:
+            raise RuntimeError(
+                "Please explicitly unset 'ARROW_PRE_0_15_IPC_FORMAT' environment variable in "
+                "both driver and executor sides. Check your spark.executorEnv.*, "
+                "spark.yarn.appMasterEnv.*, spark.mesos.driverEnv.* and "
+                "spark.kubernetes.driverEnv.* configurations. It is required to set this "
+                "environment variable only when you use pyarrow>=0.15 and pyspark<3.0."
+            )
+    return session
 
 
 def validate_arguments_and_invoke_function(
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@
     python_requires='>=3.5',
     install_requires=[
         'pandas>=0.23.2',
-        'pyarrow>=0.10,<0.15',
+        'pyarrow>=0.10',
         'numpy>=1.14',
         'matplotlib>=3.0.0',
     ],


Test patch: