Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/core/util.py b/great_expectations/core/util.py
--- a/great_expectations/core/util.py
+++ b/great_expectations/core/util.py
@@ -795,7 +795,7 @@ def sniff_s3_compression(s3_url: S3Url) -> Union[str, None]:
 # noinspection PyPep8Naming
 def get_or_create_spark_application(
     spark_config: Optional[Dict[str, str]] = None,
-    force_reuse_spark_context: bool = False,
+    force_reuse_spark_context: bool = True,
 ) -> SparkSession:
     """Obtains configured Spark session if it has already been initialized; otherwise creates Spark session, configures it, and returns it to caller.
 
@@ -809,7 +809,7 @@ def get_or_create_spark_application(
         spark_config: Dictionary containing Spark configuration (string-valued keys mapped to string-valued properties).
         force_reuse_spark_context: Boolean flag indicating (if True) that creating new Spark context is forbidden.
 
-    Returns: SparkSession (new or existing as per "isStopped()" status and "force_reuse_spark_context" directive).
+    Returns: SparkSession (new or existing as per "isStopped()" status).
     """
     if spark_config is None:
         spark_config = {}
diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py
--- a/great_expectations/execution_engine/sparkdf_execution_engine.py
+++ b/great_expectations/execution_engine/sparkdf_execution_engine.py
@@ -212,7 +212,7 @@ def __init__(
         *args,
         persist=True,
         spark_config=None,
-        force_reuse_spark_context=False,
+        force_reuse_spark_context=True,
         **kwargs,
     ) -> None:
         self._persist = persist


Test patch:
diff --git a/tests/datasource/test_new_datasource.py b/tests/datasource/test_new_datasource.py
--- a/tests/datasource/test_new_datasource.py
+++ b/tests/datasource/test_new_datasource.py
@@ -243,17 +243,16 @@ def test_basic_spark_datasource_self_check(basic_spark_datasource):
     # we deem asserting certain key-value pairs sufficient for purposes of this test
     expected_spark_config: Dict[str, str] = {
         "spark.app.name": "default_great_expectations_spark_application",
-        "spark.default.parallelism": "4",
+        "spark.default.parallelism": 4,
         "spark.driver.memory": "6g",
         "spark.executor.id": "driver",
-        "spark.executor.memory": "6g",
+        "spark.executor.memory": "450m",
         "spark.master": "local[*]",
         "spark.rdd.compress": "True",
         "spark.serializer.objectStreamReset": "100",
         "spark.sql.catalogImplementation": "hive",
-        "spark.sql.shuffle.partitions": "2",
+        "spark.sql.shuffle.partitions": 2,
         "spark.submit.deployMode": "client",
-        "spark.ui.showConsoleProgress": "False",
     }
     actual_spark_config: Dict[str, Any] = report["execution_engine"]["spark_config"]
 
diff --git a/tests/integration/spark/test_spark_config.py b/tests/integration/spark/test_spark_config.py
--- a/tests/integration/spark/test_spark_config.py
+++ b/tests/integration/spark/test_spark_config.py
@@ -3,7 +3,7 @@
 from typing import Dict, List
 
 import pytest
-from packaging.version import parse as parse_version
+from packaging.version import Version, parse as parse_version
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +26,12 @@
     )
 
 
+def test_current_pyspark_version_installed(spark_session):
+    pyspark_version: Version = parse_version(pyspark.__version__)
+    # Spark versions less than 3.0 are not supported.
+    assert pyspark_version.major >= 3, "Spark versions less than 3.0 are not supported."
+
+
 def test_spark_config_datasource(spark_session_v012):
     name: str = "great_expectations-ds-config"
     spark_config: Dict[str, str] = {
@@ -45,56 +51,36 @@ def test_spark_config_datasource(spark_session_v012):
     assert ("spark.app.name", name) in conf
     assert ("spark.sql.catalogImplementation", "hive") in conf
     assert ("spark.executor.memory", "768m") in conf
+    spark_session.sparkContext.stop()
 
 
-@pytest.mark.parametrize("databricks_runtime", [False, True])
-def test_spark_config_execution_engine(spark_session, databricks_runtime):
-    # we observe different behaviour depending on Spark version:
-    # https://spark.apache.org/docs/3.0.0/pyspark-migration-guide.html#upgrading-from-pyspark-24-to-30
-    pyspark_version = parse_version(pyspark.__version__)
-
-    # keep track of spark app id
+def test_spark_config_execution_engine(spark_session):
     old_app_id = spark_session.sparkContext.applicationId
-
-    if databricks_runtime:
-        # simulate a databricks runtime environment by setting the databricks runtime version
-        os.environ["DATABRICKS_RUNTIME_VERSION"] = "7.3"
-
-    name: str = "great_expectations-ee-config"
-    spark_config: Dict[str, str] = {
-        "spark.app.name": name,
+    new_spark_config: Dict[str, str] = {
+        "spark.app.name": "great_expectations-ee-config",
         "spark.sql.catalogImplementation": "hive",
         "spark.executor.memory": "512m",
         # "spark.driver.allowMultipleContexts": "true",  # This directive does not appear to have any effect.
     }
-    execution_engine = SparkDFExecutionEngine(spark_config=spark_config)
-    spark_session: SparkSession = execution_engine.spark
+    execution_engine = SparkDFExecutionEngine(spark_config=new_spark_config)
+    new_spark_session: SparkSession = execution_engine.spark
+
     # noinspection PyProtectedMember
-    sc_stopped: bool = spark_session.sparkContext._jsc.sc().isStopped()
+    sc_stopped: bool = new_spark_session.sparkContext._jsc.sc().isStopped()
+
     assert not sc_stopped
 
-    # Test that our values were set if not running in a Databricks runtime
-    conf: List[tuple] = execution_engine.spark.sparkContext.getConf().getAll()
-    if not databricks_runtime:
-        # we can safely "restart" the SparkContext
-        if pyspark_version.major >= 3:
-            # spark context is restarted because the "spark.app.name" is not the one desired
-            # this happens because from Spark 3, the SparkSession builder won't update config values
-            assert old_app_id != execution_engine.spark.sparkContext.applicationId
-        assert ("spark.app.name", "great_expectations-ee-config") in conf
-        assert ("spark.sql.catalogImplementation", "hive") in conf
-        assert ("spark.executor.memory", "512m") in conf
-    else:
-        # we should not "restart" the SparkContext
-        if pyspark_version.major >= 3:
-            assert old_app_id == execution_engine.spark.sparkContext.applicationId
-            # spark context config values are not changed
-            assert (
-                "spark.app.name",
-                "default_great_expectations_spark_application",
-            ) in conf
-        else:
-            # spark context config values are changed by the builder no matter what
-            assert ("spark.app.name", "great_expectations-ee-config") in conf
-            assert ("spark.sql.catalogImplementation", "hive") in conf
-            assert ("spark.executor.memory", "512m") in conf
+    current_spark_config: List[
+        tuple
+    ] = execution_engine.spark.sparkContext.getConf().getAll()
+    assert old_app_id == execution_engine.spark.sparkContext.applicationId
+    assert ("spark.sql.catalogImplementation", "hive") in current_spark_config
+    # Confirm that "spark.app.name" was not changed upon "SparkDFExecutionEngine" instantiation (from original value).
+    assert (
+        "spark.app.name",
+        "default_great_expectations_spark_application",
+    ) in current_spark_config
+    assert ("spark.executor.memory", "450m") in current_spark_config
+    # spark context config values cannot be changed by the builder no matter what
+    assert current_spark_config != new_spark_config
+    new_spark_session.sparkContext.stop()