Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/core/util.py b/great_expectations/core/util.py
--- a/great_expectations/core/util.py
+++ b/great_expectations/core/util.py
@@ -1,10 +1,11 @@
+import copy
 import datetime
 import decimal
 import logging
 import sys
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import Any, Dict, Iterable, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Union
 from urllib.parse import urlparse
 
 import numpy as np
@@ -448,10 +449,8 @@ def get_or_create_spark_application(
     # This is preferred to calling "return SparkSession.builder.getOrCreate()", which will result in the setting
     # ("spark.app.name", "pyspark-shell") remaining in SparkConf statically for the entire duration of the "pytest" run.
     try:
-        from pyspark import SparkContext
         from pyspark.sql import SparkSession
     except ImportError:
-        SparkContext = None
         SparkSession = None
         # TODO: review logging more detail here
         logger.debug(
@@ -460,6 +459,8 @@ def get_or_create_spark_application(
 
     if spark_config is None:
         spark_config = {}
+    else:
+        spark_config = copy.deepcopy(spark_config)
     name: Optional[str] = spark_config.get("spark.app.name")
     if not name:
         name = "default_great_expectations_spark_application"
@@ -471,15 +472,17 @@ def get_or_create_spark_application(
     if spark_session is None:
         raise ValueError("SparkContext could not be started.")
 
-    sc: SparkContext = spark_session.sparkContext
     # noinspection PyProtectedMember
-    sc_stopped: bool = sc._jsc.sc().isStopped()
-    if ("spark.app.name", name) not in sc.getConf().getAll():
+    sc_stopped: bool = spark_session.sparkContext._jsc.sc().isStopped()
+    if spark_restart_required(
+        current_spark_config=spark_session.sparkContext.getConf().getAll(),
+        desired_spark_config=spark_config,
+    ):
         if not sc_stopped:
             try:
                 # We need to stop the old/default Spark session in order to reconfigure it with the desired options.
                 logger.info("Stopping existing spark context to reconfigure.")
-                sc.stop()
+                spark_session.sparkContext.stop()
             except AttributeError:
                 logger.error(
                     "Unable to load spark context; install optional spark dependency for support."
@@ -487,9 +490,8 @@ def get_or_create_spark_application(
         spark_session = get_or_create_spark_session(spark_config=spark_config)
         if spark_session is None:
             raise ValueError("SparkContext could not be started.")
-        sc = spark_session.sparkContext
         # noinspection PyProtectedMember
-        sc_stopped = sc._jsc.sc().isStopped()
+        sc_stopped = spark_session.sparkContext._jsc.sc().isStopped()
 
     if sc_stopped:
         raise ValueError("SparkContext stopped unexpectedly.")
@@ -507,10 +509,8 @@ def get_or_create_spark_session(
     # This is preferred to calling "return SparkSession.builder.getOrCreate()", which will result in the setting
     # ("spark.app.name", "pyspark-shell") remaining in SparkConf statically for the entire duration of the "pytest" run.
     try:
-        from pyspark import SparkContext
         from pyspark.sql import SparkSession
     except ImportError:
-        SparkContext = None
         SparkSession = None
         # TODO: review logging more detail here
         logger.debug(
@@ -521,6 +521,8 @@ def get_or_create_spark_session(
     try:
         if spark_config is None:
             spark_config = {}
+        else:
+            spark_config = copy.deepcopy(spark_config)
 
         builder = SparkSession.builder
 
@@ -531,9 +533,8 @@ def get_or_create_spark_session(
             if k != "spark.app.name":
                 builder.config(k, v)
         spark_session = builder.getOrCreate()
-        sc: SparkContext = spark_session.sparkContext
         # noinspection PyProtectedMember
-        if sc._jsc.sc().isStopped():
+        if spark_session.sparkContext._jsc.sc().isStopped():
             raise ValueError("SparkContext stopped unexpectedly.")
     except AttributeError:
         logger.error(
@@ -544,6 +545,23 @@ def get_or_create_spark_session(
     return spark_session
 
 
+def spark_restart_required(
+    current_spark_config: List[tuple], desired_spark_config: dict
+) -> bool:
+    current_spark_config_dict: dict = {k: v for (k, v) in current_spark_config}
+    if desired_spark_config.get("spark.app.name") != current_spark_config_dict.get(
+        "spark.app.name"
+    ):
+        return True
+
+    if not set([(k, v) for k, v in desired_spark_config.items()]).issubset(
+        current_spark_config
+    ):
+        return True
+
+    return False
+
+
 def get_sql_dialect_floating_point_infinity_value(
     schema: str, negative: bool = False
 ) -> float:
diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py
--- a/great_expectations/execution_engine/sparkdf_execution_engine.py
+++ b/great_expectations/execution_engine/sparkdf_execution_engine.py
@@ -32,6 +32,7 @@
 try:
     import pyspark
     import pyspark.sql.functions as F
+    from pyspark import SparkContext
     from pyspark.sql import DataFrame, SparkSession
     from pyspark.sql.types import (
         BooleanType,
@@ -42,10 +43,9 @@
         StructField,
         StructType,
     )
-
-
 except ImportError:
     pyspark = None
+    SparkContext = None
     SparkSession = None
     DataFrame = None
     F = None
@@ -150,8 +150,12 @@ def __init__(self, *args, persist=True, spark_config=None, **kwargs):
             spark_config = {}
 
         spark: SparkSession = get_or_create_spark_application(spark_config=spark_config)
-        self.spark = spark
+
+        spark_config = dict(spark_config)
+        spark_config.update({k: v for (k, v) in spark.sparkContext.getConf().getAll()})
+
         self._spark_config = spark_config
+        self.spark = spark
 
         super().__init__(*args, **kwargs)
 


Test patch:
diff --git a/tests/datasource/test_new_datasource.py b/tests/datasource/test_new_datasource.py
--- a/tests/datasource/test_new_datasource.py
+++ b/tests/datasource/test_new_datasource.py
@@ -197,6 +197,10 @@ def test_basic_pandas_datasource_v013_self_check(basic_pandas_datasource_v013):
 
 def test_basic_spark_datasource_self_check(basic_spark_datasource):
     report = basic_spark_datasource.self_check()
+    report["execution_engine"]["spark_config"].pop("spark.app.id", None)
+    report["execution_engine"]["spark_config"].pop("spark.driver.host", None)
+    report["execution_engine"]["spark_config"].pop("spark.driver.port", None)
+    report["execution_engine"]["spark_config"].pop("spark.submit.pyFiles", None)
     assert report == {
         "execution_engine": {
             "caching": True,
@@ -206,11 +210,16 @@ def test_basic_spark_datasource_self_check(basic_spark_datasource):
             "spark_config": {
                 "spark.app.name": "default_great_expectations_spark_application",
                 "spark.master": "local[*]",
+                "spark.executor.id": "driver",
                 "spark.executor.memory": "6g",
                 "spark.driver.memory": "6g",
-                "spark.ui.showConsoleProgress": False,
-                "spark.sql.shuffle.partitions": 2,
-                "spark.default.parallelism": 4,
+                "spark.ui.showConsoleProgress": "False",
+                "spark.serializer.objectStreamReset": "100",
+                "spark.sql.catalogImplementation": "hive",
+                "spark.sql.shuffle.partitions": "2",
+                "spark.default.parallelism": "4",
+                "spark.rdd.compress": "True",
+                "spark.submit.deployMode": "client",
             },
         },
         "data_connectors": {
diff --git a/tests/integration/spark/test_spark_config.py b/tests/integration/spark/test_spark_config.py
--- a/tests/integration/spark/test_spark_config.py
+++ b/tests/integration/spark/test_spark_config.py
@@ -31,10 +31,9 @@ def test_spark_config_datasource(spark_session_v012):
         # "spark.driver.allowMultipleContexts": "true",  # This directive does not appear to have any effect.
     }
     source: SparkDFDatasource = SparkDFDatasource(spark_config=spark_config)
-    sess: SparkSession = source.spark
-    sc: SparkContext = sess.sparkContext
+    spark_session: SparkSession = source.spark
     # noinspection PyProtectedMember
-    sc_stopped: bool = sc._jsc.sc().isStopped()
+    sc_stopped: bool = spark_session.sparkContext._jsc.sc().isStopped()
     assert not sc_stopped
 
     # Test that our values were set
@@ -55,10 +54,9 @@ def test_spark_config_execution_engine(spark_session):
     execution_engine: SparkDFExecutionEngine = SparkDFExecutionEngine(
         spark_config=spark_config
     )
-    sess: SparkSession = execution_engine.spark
-    sc: SparkContext = sess.sparkContext
+    spark_session: SparkSession = execution_engine.spark
     # noinspection PyProtectedMember
-    sc_stopped: bool = sc._jsc.sc().isStopped()
+    sc_stopped: bool = spark_session.sparkContext._jsc.sc().isStopped()
     assert not sc_stopped
 
     # Test that our values were set