Repository: OpenLineage/OpenLineage (2025-07-22T00:58:49Z)

Issue:
[Spark4] AlterTableCommandDatasetBuilder turned off for Spark 4 integration
`AlterTableCommandDatasetBuilder` has been disabled for Spark4 in `Spark40DatasetBuilderFactory` class because of: 
```
java.lang.NoClassDefFoundError: org/apache/spark/sql/catalyst/plans/logical/AlterColumn
	at io.openlineage.spark32.agent.lifecycle.plan.AlterTableCommandDatasetBuilder.isDefinedAtLogicalPlan(AlterTableCommandDatasetBuilder.java:51) ~[openlineage-spark-agent_2.13-1.35.0-SNAPSHOT-shadow.jar:1.35.0-SNAPSHOT]
	at io.openlineage.spark.api.AbstractQueryPlanOutputDatasetBuilder.lambda$jobNameSuffixFromLogicalPlan$3(AbstractQueryPlanOutputDatasetBuilder.java:76) ~[openlineage-spark-agent_2.13-1.35.0-SNAPSHOT-shadow.jar:1.35.0-SNAPSHOT]
	at java.base/java.util.Optional.filter(Optional.java:218) ~[?:?]
	at io.openlineage.spark.api.AbstractQueryPlanOutputDatasetBuilder.jobNameSuffixFromLogicalPlan(AbstractQueryPlanOutputDatasetBuilder.java:76) ~[openlineage-spark-agent_2.13-1.35.0-SNAPSHOT-shadow.jar:1.35.0-SNAPSHOT]

```

Solution: prepare new `AlterTableCommandDatasetBuilder` in spark40 package.

This issue if a follow-up of https://github.com/OpenLineage/OpenLineage/pull/3877


Discussion:
picking this up. thanks!



Patch:
diff --git a/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/Spark40DatasetBuilderFactory.java b/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/Spark40DatasetBuilderFactory.java
index 68654e3812..f7b88508c1 100644
--- a/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/Spark40DatasetBuilderFactory.java
+++ b/integration/spark/app/src/main/java/io/openlineage/spark/agent/lifecycle/Spark40DatasetBuilderFactory.java
@@ -37,6 +37,7 @@
 import io.openlineage.spark34.agent.lifecycle.plan.column.DropTableDatasetBuilder;
 import io.openlineage.spark35.agent.lifecycle.plan.CreateReplaceOutputDatasetBuilder;
 import io.openlineage.spark35.agent.lifecycle.plan.MergeRowsColumnLineageVisitor;
+import io.openlineage.spark40.agent.lifecycle.plan.AlterTableCommandDatasetBuilder;
 import io.openlineage.spark40.agent.lifecycle.plan.StreamingDataSourceV2ScanRelationDatasetBuilder;
 import java.util.Collection;
 import java.util.List;
@@ -88,10 +89,8 @@ public Collection<PartialFunction<Object, List<OpenLineage.OutputDataset>>> getO
             .add(new SparkExtensionV1OutputDatasetBuilder(context))
             .add(new SubqueryAliasOutputDatasetBuilder(context))
             .add(new DropTableDatasetBuilder(context))
-            .add(new MergeIntoCommandEdgeOutputDatasetBuilder(context));
-    // .add(new AlterTableCommandDatasetBuilder(context); one of the classes no longer present for
-    // Spark 4
-    // TODO: https://github.com/OpenLineage/OpenLineage/issues/3884
+            .add(new MergeIntoCommandEdgeOutputDatasetBuilder(context))
+            .add(new AlterTableCommandDatasetBuilder(context));
 
     if (ReplaceIcebergDataDatasetBuilder.hasClasses()) {
       builder.add(new ReplaceIcebergDataDatasetBuilder(context));
diff --git a/integration/spark/spark40/src/main/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilder.java b/integration/spark/spark40/src/main/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilder.java
new file mode 100644
index 0000000000..e0e507d662
--- /dev/null
+++ b/integration/spark/spark40/src/main/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilder.java
@@ -0,0 +1,111 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark40.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.dataset.DatasetCompositeFacetsBuilder;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.spark.agent.util.PlanUtils;
+import io.openlineage.spark.api.AbstractQueryPlanOutputDatasetBuilder;
+import io.openlineage.spark.api.OpenLineageContext;
+import io.openlineage.spark3.agent.lifecycle.plan.catalog.CatalogUtils3;
+import io.openlineage.spark3.agent.utils.PlanUtils3;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.scheduler.SparkListenerEvent;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.analysis.ResolvedTable;
+import org.apache.spark.sql.catalyst.plans.logical.AddColumns;
+import org.apache.spark.sql.catalyst.plans.logical.AlterTableCommand;
+import org.apache.spark.sql.catalyst.plans.logical.CommentOnTable;
+import org.apache.spark.sql.catalyst.plans.logical.DropColumns;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.catalyst.plans.logical.RenameColumn;
+import org.apache.spark.sql.catalyst.plans.logical.ReplaceColumns;
+import org.apache.spark.sql.catalyst.plans.logical.SetTableLocation;
+import org.apache.spark.sql.catalyst.plans.logical.SetTableProperties;
+import org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.Table;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.types.StructType;
+
+@Slf4j
+public class AlterTableCommandDatasetBuilder
+    extends AbstractQueryPlanOutputDatasetBuilder<LogicalPlan> {
+
+  public AlterTableCommandDatasetBuilder(@NonNull OpenLineageContext context) {
+    super(context, false);
+  }
+
+  @Override
+  public boolean isDefinedAtLogicalPlan(LogicalPlan x) {
+    return x instanceof CommentOnTable
+        || x instanceof SetTableLocation
+        || x instanceof SetTableProperties
+        || x instanceof UnsetTableProperties
+        || x instanceof AddColumns
+        || x instanceof ReplaceColumns
+        || x instanceof DropColumns
+        || x instanceof RenameColumn;
+    // AlterColumn removed for Spark 4.0 compatibility
+  }
+
+  @Override
+  protected List<OpenLineage.OutputDataset> apply(
+      SparkListenerEvent event, LogicalPlan alterTableCommand) {
+    ResolvedTable resolvedTable = (ResolvedTable) ((AlterTableCommand) alterTableCommand).table();
+    Table table;
+    try {
+      // resolvedTable has only old metadata (before alter)
+      table = resolvedTable.catalog().loadTable(resolvedTable.identifier());
+    } catch (NoSuchTableException e) {
+      return Collections.emptyList();
+    }
+    TableCatalog tableCatalog = resolvedTable.catalog();
+    Map<String, String> tableProperties = table.properties();
+    Identifier identifier = resolvedTable.identifier();
+    StructType schema = table.schema();
+    OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange lifecycleStateChange =
+        OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange.ALTER;
+
+    Optional<DatasetIdentifier> di =
+        PlanUtils3.getDatasetIdentifier(context, tableCatalog, identifier, tableProperties);
+
+    if (!di.isPresent()) {
+      return Collections.emptyList();
+    }
+
+    OpenLineage openLineage = context.getOpenLineage();
+    DatasetCompositeFacetsBuilder builder = new DatasetCompositeFacetsBuilder(openLineage);
+    builder
+        .getFacets()
+        .schema(PlanUtils.schemaFacet(openLineage, schema))
+        .lifecycleStateChange(
+            openLineage.newLifecycleStateChangeDatasetFacet(lifecycleStateChange, null))
+        .dataSource(PlanUtils.datasourceFacet(openLineage, di.get().getNamespace()));
+
+    if (includeDatasetVersion(event)) {
+      Optional<String> datasetVersion =
+          CatalogUtils3.getDatasetVersion(
+              context, resolvedTable.catalog(), resolvedTable.identifier(), table.properties());
+      datasetVersion.ifPresent(
+          version ->
+              builder.getFacets().version(openLineage.newDatasetVersionDatasetFacet(version)));
+    }
+    return Collections.singletonList(outputDataset().getDataset(di.get(), builder));
+  }
+
+  @Override
+  public Optional<String> jobNameSuffix(LogicalPlan alterTableCommand) {
+    ResolvedTable resolvedTable = (ResolvedTable) ((AlterTableCommand) alterTableCommand).table();
+    return Optional.of(identToSuffix(resolvedTable.identifier()));
+  }
+}


Test patch:
diff --git a/integration/spark/spark40/src/test/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilderTest.java b/integration/spark/spark40/src/test/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilderTest.java
new file mode 100644
index 0000000000..a01f742773
--- /dev/null
+++ b/integration/spark/spark40/src/test/java/io/openlineage/spark40/agent/lifecycle/plan/AlterTableCommandDatasetBuilderTest.java
@@ -0,0 +1,148 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark40.agent.lifecycle.plan;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.spark.agent.Versions;
+import io.openlineage.spark.api.OpenLineageContext;
+import io.openlineage.spark.api.SparkOpenLineageConfig;
+import io.openlineage.spark3.agent.lifecycle.plan.catalog.CatalogUtils3;
+import io.openlineage.spark3.agent.utils.PlanUtils3;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import lombok.SneakyThrows;
+import org.apache.spark.SparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+import org.apache.spark.sql.catalyst.analysis.ResolvedTable;
+import org.apache.spark.sql.catalyst.plans.logical.CommentOnTable;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.Table;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import scala.Option;
+
+class AlterTableCommandDatasetBuilderTest {
+
+  OpenLineageContext openLineageContext =
+      OpenLineageContext.builder()
+          .sparkSession(mock(SparkSession.class))
+          .sparkContext(mock(SparkContext.class))
+          .openLineage(new OpenLineage(Versions.OPEN_LINEAGE_PRODUCER_URI))
+          .meterRegistry(new SimpleMeterRegistry())
+          .openLineageConfig(new SparkOpenLineageConfig())
+          .build();
+
+  TableCatalog tableCatalog = mock(TableCatalog.class);
+  StructType schema = new StructType();
+  Table table = mock(Table.class);
+  Map<String, String> tableProperties = new HashMap<>();
+  DatasetIdentifier di = new DatasetIdentifier("table", "db");
+  Identifier identifier = mock(Identifier.class);
+  CommentOnTable alterTable = mock(CommentOnTable.class);
+
+  ResolvedTable resolvedTable = mock(ResolvedTable.class);
+
+  AlterTableCommandDatasetBuilder builder = new AlterTableCommandDatasetBuilder(openLineageContext);
+
+  @BeforeEach
+  public void setUp() {
+    when(alterTable.table()).thenReturn(resolvedTable);
+    when(resolvedTable.catalog()).thenReturn(tableCatalog);
+    when(resolvedTable.identifier()).thenReturn(identifier);
+    when(resolvedTable.table()).thenReturn(table);
+    when(resolvedTable.table().schema()).thenReturn(schema);
+    when(resolvedTable.table().properties()).thenReturn(tableProperties);
+  }
+
+  @Test
+  @SneakyThrows
+  void testApplyWhenTableNotFound() {
+    when(tableCatalog.loadTable(identifier)).thenThrow(mock(NoSuchTableException.class));
+    List<OpenLineage.OutputDataset> outputDatasets =
+        builder.apply(new SparkListenerSQLExecutionEnd(1L, 1L, Option.<String>empty()), alterTable);
+    assertEquals(0, outputDatasets.size());
+  }
+
+  @Test
+  @SneakyThrows
+  void testApplyWhenNoDatasetIdentifier() {
+    when(tableCatalog.loadTable(identifier)).thenReturn(table);
+    try (MockedStatic mocked = mockStatic(PlanUtils3.class)) {
+      when(PlanUtils3.getDatasetIdentifier(
+              openLineageContext, tableCatalog, identifier, tableProperties))
+          .thenReturn(Optional.empty());
+
+      List<OpenLineage.OutputDataset> outputDatasets =
+          builder.apply(
+              new SparkListenerSQLExecutionEnd(1L, 1L, Option.<String>empty()), alterTable);
+      assertEquals(0, outputDatasets.size());
+    }
+  }
+
+  @Test
+  @SneakyThrows
+  void testApply() {
+    when(tableCatalog.loadTable(identifier)).thenReturn(table);
+    try (MockedStatic mocked = mockStatic(PlanUtils3.class)) {
+      when(PlanUtils3.getDatasetIdentifier(
+              openLineageContext, tableCatalog, identifier, tableProperties))
+          .thenReturn(Optional.of(di));
+
+      List<OpenLineage.OutputDataset> outputDatasets =
+          builder.apply(
+              new SparkListenerSQLExecutionEnd(1L, 1L, Option.<String>empty()), alterTable);
+
+      assertEquals(1, outputDatasets.size());
+      assertEquals("table", outputDatasets.get(0).getName());
+      assertEquals("db", outputDatasets.get(0).getNamespace());
+    }
+  }
+
+  @Test
+  @SneakyThrows
+  void testApplyDatasetVersionIncluded() {
+    when(tableCatalog.loadTable(identifier)).thenReturn(table);
+    try (MockedStatic mocked = mockStatic(PlanUtils3.class)) {
+      try (MockedStatic mockCatalog = mockStatic(CatalogUtils3.class)) {
+        when(CatalogUtils3.getDatasetVersion(
+                openLineageContext, tableCatalog, identifier, tableProperties))
+            .thenReturn(Optional.of("v2"));
+
+        when(PlanUtils3.getDatasetIdentifier(
+                openLineageContext, tableCatalog, identifier, tableProperties))
+            .thenReturn(Optional.of(di));
+
+        List<OpenLineage.OutputDataset> outputDatasets =
+            builder.apply(
+                new SparkListenerSQLExecutionEnd(1L, 1L, Option.<String>empty()), alterTable);
+
+        assertEquals(1, outputDatasets.size());
+        assertEquals("v2", outputDatasets.get(0).getFacets().getVersion().getDatasetVersion());
+      }
+    }
+  }
+
+  @Test
+  void testJobNameSuffix() {
+    when(resolvedTable.identifier()).thenReturn(Identifier.of(new String[] {"a", "b"}, "c"));
+    assertThat(builder.jobNameSuffix(alterTable).get()).isEqualTo("a_b_c");
+  }
+}
diff --git a/integration/spark/spark40/src/test/resources/io/openlineage/spark/agent/version.properties b/integration/spark/spark40/src/test/resources/io/openlineage/spark/agent/version.properties
new file mode 100644
index 0000000000..2c1ba5ea06
--- /dev/null
+++ b/integration/spark/spark40/src/test/resources/io/openlineage/spark/agent/version.properties
@@ -0,0 +1,1 @@
+version 1.36.0-SNAPSHOT
\ No newline at end of file