Repository: spetlr-org/spetlr (language: python)

Issue:
End support for LTS 9.1
Let's end support for LTS 9.1 and also for python 3.8. Lowest supported version will then be 10.4. Please leave a comment if you agree.


Discussion / hints:
Agree. 
The end of support for Databricks runtime 9.1 LTS will end on the 15th of September 2023. Please leave a comment if you still need support after that. 

Fix patch:
diff --git a/.github/deploy-bicep/resources-integration.bicep b/.github/deploy-bicep/resources-integration.bicep
index d3b9648e..2508ab63 100644
--- a/.github/deploy-bicep/resources-integration.bicep
+++ b/.github/deploy-bicep/resources-integration.bicep
@@ -102,7 +102,7 @@ resource eventhubs 'Microsoft.EventHub/namespaces@2021-11-01' = {
             storageAccountResourceId: staccount.id
           }
         }
-        skipEmptyArchives: true
+        skipEmptyArchives: false
         encoding: 'Avro'
       }
     }
diff --git a/.github/submit/sparklibs91.json b/.github/submit/sparklibs91.json
deleted file mode 100644
index 4713123b..00000000
--- a/.github/submit/sparklibs91.json
+++ /dev/null
@@ -1,17 +0,0 @@
-[
-  {
-    "maven" : {
-      "coordinates" : "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21"
-    }
-  },
-  {
-    "maven" : {
-      "coordinates" : "com.microsoft.azure:spark-mssql-connector_2.12_3.0:1.0.0-alpha"
-    }
-  },
-  {
-    "maven" : {
-      "coordinates" : "com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.0.0-beta.3"
-    }
-  }
-]
diff --git a/.github/workflows/pre-integration.yml b/.github/workflows/pre-integration.yml
index ad612c55..5dfd108a 100644
--- a/.github/workflows/pre-integration.yml
+++ b/.github/workflows/pre-integration.yml
@@ -76,16 +76,6 @@ jobs:
         shell: pwsh
         run: Start-Sleep -s 30
 
-
-      - name: Launch integration tests 9.1
-        shell: pwsh
-        run: |
-          .github/submit/submit_test_job.ps1  `
-            -sparkVersion "9.1.x-scala2.12" `
-            -testJobDetails job91.json `
-            -sparkLibs sparklibs91.json
-
-
       - name: Launch integration tests 10.4
         shell: pwsh
         run: |
@@ -106,10 +96,6 @@ jobs:
         shell: pwsh
         run: Start-Sleep -s 120
 
-      - name: Fetch integration tests 9.1
-        shell: pwsh
-        run: .github/submit/fetch_test_job.ps1 -testJobDetails job91.json
-
       - name: Fetch integration tests 10.4
         shell: pwsh
         run: .github/submit/fetch_test_job.ps1 -testJobDetails job104.json
diff --git a/.gitignore b/.gitignore
index 411c78d3..69ab1c10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -140,3 +140,4 @@ dmypy.json
 # environment dependent files
 .dbx/project.json
 tests/cluster/mount/mounts.json
+.DS_Store


Test patch:
diff --git a/tests/cluster/eh/test_eh_saving.py b/tests/cluster/eh/test_eh_saving.py
index 927d691b..a751e089 100644
--- a/tests/cluster/eh/test_eh_saving.py
+++ b/tests/cluster/eh/test_eh_saving.py
@@ -3,15 +3,15 @@
 from datetime import datetime, timedelta, timezone
 
 from pyspark.sql import DataFrame
-from pyspark.sql import functions as f
 from spetlrtools.time import dt_utc
 
 from spetlr import Configurator
 from spetlr.delta import DeltaHandle
-from spetlr.eh import EventHubCapture, EventHubJsonPublisher
+from spetlr.eh import EventHubJsonPublisher
 from spetlr.eh.EventHubCaptureExtractor import EventHubCaptureExtractor
 from spetlr.etl import Transformer
-from spetlr.functions import init_dbutils
+
+# from spetlr.functions import init_dbutils
 from spetlr.orchestrators import EhJsonToDeltaOrchestrator
 from spetlr.spark import Spark
 from tests.cluster.values import resourceName
@@ -24,54 +24,15 @@ class EventHubsTests(unittest.TestCase):
     def setUpClass(cls) -> None:
         Configurator().clear_all_configurations()
 
-    def test_01_publish(self):
+    def test_01_publish_and_read(self):
         eh = SpetlrEh()
 
         df = Spark.get().createDataFrame([(1, "a"), (2, "b")], "id int, name string")
         publisher = EventHubJsonPublisher(eh)
         publisher.save(df)
 
-    def test_02_wait_for_capture_files(self):
-        # wait until capture file appears
-        dbutils = init_dbutils()
-
-        limit = datetime.now() + timedelta(minutes=10)
-
-        while datetime.now() < limit:
-            conts = {
-                item.name for item in dbutils.fs.ls(f"/mnt/{resourceName()}/silver")
-            }
-            if f"{resourceName()}/" in conts:
-                break
-            else:
-                time.sleep(10)
-                continue
-        else:
-            self.assertTrue(False, "The capture file never appeared.")
-
-        self.assertTrue(True, "The capture file has appeared.")
-
-    def test_03_read_eh_capture(self):
-        tc = Configurator()
-        tc.register(
-            "SpetlrEh",
-            {
-                "name": "SpetlrEh",
-                "path": f"/mnt/{resourceName()}/silver/{resourceName()}/spetlreh",
-                "format": "avro",
-                "partitioning": "ymd",
-            },
-        )
-        eh = EventHubCapture.from_tc("SpetlrEh")
-        df = eh.read()
-
-        df = df.select(f.from_json("body", "id int, name string").alias("body")).select(
-            "body.*"
-        )
-        rows = {tuple(row) for row in df.collect()}
-        self.assertEqual({(1, "a"), (2, "b")}, rows)
+        time.sleep(100)  # just wait the EH captures once a minute anyway.
 
-    def test_04_read_eh_capture_extractor(self):
         tc = Configurator()
         tc.register(
             "SpetlrEh",
@@ -103,7 +64,7 @@ def test_04_read_eh_capture_extractor(self):
             abs((row_written.astimezone(timezone.utc) - dt_utc()).total_seconds()), 1000
         )
 
-    def test_05_eh_json_orchestrator(self):
+        # def test_05_eh_json_orchestrator(self):
         # the orchestrator has a complex functionality that can only be fully tested
         # on a substantial holding of capture files. That is not possible here, but
         # such tests were carried out during development.
diff --git a/tests/cluster/mount/setup_job.yml.j2 b/tests/cluster/mount/setup_job.yml.j2
index a8d4151f..4e1fa6b8 100644
--- a/tests/cluster/mount/setup_job.yml.j2
+++ b/tests/cluster/mount/setup_job.yml.j2
@@ -11,7 +11,7 @@ custom:
     spot_bid_max_price: -1
 
   basic-cluster-single: &basic-cluster-single
-    spark_version: "9.1.x-scala2.12"
+    spark_version: "10.4.x-scala2.12"
     spark_conf: *spark-conf-single
     azure_attributes: *azure-attributes
     spark_env_vars: