Repository: aehrc/pathling (language: python)

Issue:
Error resolving coding literal among columns
Sentry Issue: [PATHLING-4T](https://health-informatics.sentry.io/issues/4150650588/?referrer=github_integration)

To reproduce, issue the following extract request:

```
GET /fhir/DiagnosticReport/$extract?column=result.resolve().code.coding.system&column=http%3A%2F%2Fsnomed.info%2Fsct%7C372823004
```

This produces the following error:
```
AnalysisException: Cannot resolve column name "struct(NULL AS id, http://snomed.info/sct AS system, NULL AS version, 372823004 AS code, NULL AS display, NULL AS userSelected, NULL AS _fid)" among (_jqwh9e, struct(NULL AS id, http://snomed.info/sct AS system, NULL AS version, 372823004 AS code, NULL AS display, NULL AS userSelected, NULL AS _fid)); did you mean to quote the `struct(NULL AS id, http://snomed.info/sct AS system, NULL AS version, 372823004 AS code, NULL AS display, NULL AS userSelected, NULL AS _fid)` column?
    at org.apache.spark.sql.errors.QueryCompilationErrors$.cannotResolveColumnNameAmongFieldsError(QueryCompilationErrors.scala:2265)
    at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$resolveException(Dataset.scala:259)
    at org.apache.spark.sql.Dataset.$anonfun$resolve$1(Dataset.scala:251)
    at scala.Option.getOrElse(Option.scala:189)
    at org.apache.spark.sql.Dataset.resolve(Dataset.scala:251)
...
(28 additional frame(s) were not displayed)
```


Discussion / hints:

Fix patch:
diff --git a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java
index 1720e9a8fa..f92f119d11 100644
--- a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java
+++ b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/FhirPath.java
@@ -77,6 +77,7 @@ public interface FhirPath extends Orderable {
 
   /**
    * Trims the columns to those common with the target and sorts them, ready for a union operation.
+   * The value column of the path is always included as the last column of the dataset.
    *
    * @param target the expression that this path will be combined with
    * @return a new {@link Dataset} with a subset of columns
diff --git a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java
index 62d3c6819c..9f04844348 100644
--- a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java
+++ b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/literal/LiteralPath.java
@@ -18,6 +18,7 @@
 package au.csiro.pathling.fhirpath.literal;
 
 import static au.csiro.pathling.QueryHelpers.getUnionableColumns;
+import static au.csiro.pathling.utilities.Strings.randomAlias;
 
 import au.csiro.pathling.errors.InvalidUserInputError;
 import au.csiro.pathling.fhirpath.FhirPath;
@@ -103,23 +104,24 @@ public abstract class LiteralPath<ValueType extends Type> implements FhirPath {
 
   @Nonnull
   protected final Optional<String> expression;
-
-  protected LiteralPath(@Nonnull final Dataset<Row> dataset, @Nonnull final Column idColumn,
-      @Nonnull final ValueType value) {
+  
+  private LiteralPath(@Nonnull final Dataset<Row> dataset, @Nonnull final Column idColumn,
+      @Nonnull final ValueType value, @Nonnull final Optional<String> expression) {
     this.idColumn = idColumn;
     this.value = value;
     this.dataset = dataset;
-    this.valueColumn = buildValueColumn();
-    this.expression = Optional.empty();
+    this.valueColumn = buildValueColumn().alias(randomAlias());
+    this.expression = expression;
+  }
+
+  protected LiteralPath(@Nonnull final Dataset<Row> dataset, @Nonnull final Column idColumn,
+      @Nonnull final ValueType value) {
+    this(dataset, idColumn, value, Optional.empty());
   }
 
   protected LiteralPath(@Nonnull final Dataset<Row> dataset, @Nonnull final Column idColumn,
       @Nonnull final ValueType value, @Nonnull final String expression) {
-    this.idColumn = idColumn;
-    this.value = value;
-    this.dataset = dataset;
-    this.valueColumn = buildValueColumn();
-    this.expression = Optional.of(expression);
+    this(dataset, idColumn, value, Optional.of(expression));
   }
 
   /**
diff --git a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/operator/CombineOperator.java b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/operator/CombineOperator.java
index a09b126b3c..9eb878d1cf 100644
--- a/fhirpath/src/main/java/au/csiro/pathling/fhirpath/operator/CombineOperator.java
+++ b/fhirpath/src/main/java/au/csiro/pathling/fhirpath/operator/CombineOperator.java
@@ -50,13 +50,12 @@ public FhirPath invoke(@Nonnull final OperatorInput input) {
 
     final Dataset<Row> leftTrimmed = left.getUnionableDataset(right);
     final Dataset<Row> rightTrimmed = right.getUnionableDataset(left);
-    final int valueColumnIndex = Arrays.asList(leftTrimmed.columns())
-        .indexOf(left.getValueColumn().toString());
-
+    // the value column is always the last column
+    final int valueColumnIndex = leftTrimmed.columns().length - 1;
     final Dataset<Row> dataset = leftTrimmed.union(rightTrimmed);
-    final String columnName = dataset.columns()[valueColumnIndex];
+    final String valueColumnName = dataset.columns()[valueColumnIndex];
     final DatasetWithColumn datasetWithColumn = createColumn(dataset,
-        dataset.col("`" + columnName + "`"));
+        dataset.col(valueColumnName));
     final Optional<Column> eidColumn = Optional.of(array(monotonically_increasing_id()));
     final Optional<Column> thisColumn = left instanceof NonLiteralPath
                                         ? ((NonLiteralPath) left).getThisColumn()


Test patch:
diff --git a/fhir-server/src/test/java/au/csiro/pathling/extract/ExtractQueryTest.java b/fhir-server/src/test/java/au/csiro/pathling/extract/ExtractQueryTest.java
index af2e70b8a2..e654d1def5 100644
--- a/fhir-server/src/test/java/au/csiro/pathling/extract/ExtractQueryTest.java
+++ b/fhir-server/src/test/java/au/csiro/pathling/extract/ExtractQueryTest.java
@@ -184,6 +184,23 @@ void literalColumn() {
         .hasRows(spark, "responses/ExtractQueryTest/literalColumn.csv");
   }
 
+  @Test
+  void resolveAndCodingLiteralColumn() {
+    subjectResource = ResourceType.DIAGNOSTICREPORT;
+    mockResource(ResourceType.DIAGNOSTICREPORT, ResourceType.OBSERVATION);
+
+    final ExtractRequest request = new ExtractRequestBuilder(subjectResource)
+        .withColumn("result.resolve().code.coding.system")
+        .withColumn("http://snomed.info/sct|372823004")
+        .withLimit(10)
+        .build();
+
+    final Dataset<Row> result = executor.buildQuery(request);
+    assertThat(result)
+        .debugAllRows()
+        .hasRows(spark, "responses/ExtractQueryTest/resolveAndCodingLiteralColumn.csv");
+  }
+
   @Test
   void codingColumn() {
     subjectResource = ResourceType.CONDITION;
diff --git a/fhir-server/src/test/resources/responses/ExtractQueryTest/resolveAndCodingLiteralColumn.csv b/fhir-server/src/test/resources/responses/ExtractQueryTest/resolveAndCodingLiteralColumn.csv
new file mode 100644
index 0000000000..2638c04650
--- /dev/null
+++ b/fhir-server/src/test/resources/responses/ExtractQueryTest/resolveAndCodingLiteralColumn.csv
@@ -0,0 +1,10 @@
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004
+http://loinc.org,http://snomed.info/sct|372823004