Repository: trinodb/trino (Java)
Date: 2020-04-15 08:11:49+00:00
Task type: Bug Fix

Issue:
Reading data after minor compaction fails in full ACID tables
Such reads fails with: "io.prestosql.spi.PrestoException: Error opening Hive split hdfs://hadoop-master:9000/user/hive/warehouse/test2/delta_0000001_0000002/bucket_00000 (offset=0, length=16054): rowsInRowGroup must be greater than zero"

This happens because hive compaction is writing orc files without row group info while Presto has a check to ensure that row group info should be present. We can remove that check for the fix.


Issue discussion:

Solution patch:
diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java b/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java
index b9dbaa93b4e9..568fea0553f9 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java
@@ -182,7 +182,7 @@ private OrcReader(
         this.rootColumn = createOrcColumn("", "", new OrcColumnId(0), footer.getTypes(), orcDataSource.getId());
 
         validateWrite(validation -> validation.getColumnNames().equals(getColumnNames()), "Unexpected column names");
-        validateWrite(validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group");
+        validateWrite(validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup().orElse(0), "Unexpected rows in group");
         if (writeValidation.isPresent()) {
             writeValidation.get().validateMetadata(orcDataSource.getId(), footer.getUserMetadata());
             writeValidation.get().validateFileStatistics(orcDataSource.getId(), footer.getFileStats());
diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java b/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java
index 084486f573bf..edc60b4466df 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java
@@ -51,6 +51,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -129,7 +130,7 @@ public OrcRecordReader(
             long splitLength,
             ColumnMetadata<OrcType> orcTypes,
             Optional<OrcDecompressor> decompressor,
-            int rowsInRowGroup,
+            OptionalInt rowsInRowGroup,
             DateTimeZone hiveStorageTimeZone,
             HiveWriterVersion hiveWriterVersion,
             MetadataReader metadataReader,
@@ -167,9 +168,6 @@ public OrcRecordReader(
         requireNonNull(options, "options is null");
         this.maxBlockBytes = options.getMaxBlockSize().toBytes();
 
-        // it is possible that old versions of orc use 0 to mean there are no row groups
-        checkArgument(rowsInRowGroup > 0, "rowsInRowGroup must be greater than zero");
-
         // sort stripes by file position
         List<StripeInfo> stripeInfos = new ArrayList<>();
         for (int i = 0; i < fileStripes.size(); i++) {
diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java b/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java
index 5315a8d4ec8f..b742e0db4f45 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java
@@ -57,6 +57,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -481,7 +482,7 @@ private List<OrcDataOutput> bufferFileFooter()
 
         Footer footer = new Footer(
                 numberOfRows,
-                rowGroupMaxRowCount,
+                rowGroupMaxRowCount == 0 ? OptionalInt.empty() : OptionalInt.of(rowGroupMaxRowCount),
                 closedStripes.stream()
                         .map(ClosedStripe::getStripeInformation)
                         .collect(toImmutableList()),
diff --git a/presto-orc/src/main/java/io/prestosql/orc/StripeReader.java b/presto-orc/src/main/java/io/prestosql/orc/StripeReader.java
index 449b4fec9b80..7a8bc17fdb1b 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/StripeReader.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/StripeReader.java
@@ -56,6 +56,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.Set;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -83,7 +84,7 @@ public class StripeReader
     private final ColumnMetadata<OrcType> types;
     private final HiveWriterVersion hiveWriterVersion;
     private final Set<OrcColumnId> includedOrcColumnIds;
-    private final int rowsInRowGroup;
+    private final OptionalInt rowsInRowGroup;
     private final OrcPredicate predicate;
     private final MetadataReader metadataReader;
     private final Optional<OrcWriteValidation> writeValidation;
@@ -94,7 +95,7 @@ public StripeReader(
             Optional<OrcDecompressor> decompressor,
             ColumnMetadata<OrcType> types,
             Set<OrcColumn> readColumns,
-            int rowsInRowGroup,
+            OptionalInt rowsInRowGroup,
             OrcPredicate predicate,
             HiveWriterVersion hiveWriterVersion,
             MetadataReader metadataReader,
@@ -133,7 +134,7 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste
 
         // handle stripes with more than one row group
         boolean invalidCheckPoint = false;
-        if (stripe.getNumberOfRows() > rowsInRowGroup) {
+        if (rowsInRowGroup.isPresent() && stripe.getNumberOfRows() > rowsInRowGroup.getAsInt()) {
             // determine ranges of the stripe to read
             Map<StreamId, DiskRange> diskRanges = getDiskRanges(stripeFooter.getStreams());
             diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet()));
@@ -332,6 +333,7 @@ private List<RowGroup> createRowGroups(
             ColumnMetadata<ColumnEncoding> encodings)
             throws InvalidCheckpointException
     {
+        int rowsInRowGroup = this.rowsInRowGroup.orElseThrow(() -> new IllegalStateException("Cannot create row groups if row group info is missing"));
         ImmutableList.Builder<RowGroup> rowGroupBuilder = ImmutableList.builder();
 
         for (int rowGroupId : selectedRowGroups) {
@@ -438,6 +440,8 @@ private Map<StreamId, List<RowGroupIndex>> readColumnIndexes(Map<StreamId, Strea
 
     private Set<Integer> selectRowGroups(StripeInformation stripe, Map<StreamId, List<RowGroupIndex>> columnIndexes)
     {
+        int rowsInRowGroup = this.rowsInRowGroup.orElseThrow(() -> new IllegalStateException("Cannot create row groups if row group info is missing"));
+
         int rowsInStripe = stripe.getNumberOfRows();
         int groupsInStripe = ceil(rowsInStripe, rowsInRowGroup);
 
diff --git a/presto-orc/src/main/java/io/prestosql/orc/metadata/Footer.java b/presto-orc/src/main/java/io/prestosql/orc/metadata/Footer.java
index 54c81c115343..f4399edf6457 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/metadata/Footer.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/metadata/Footer.java
@@ -22,15 +22,17 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalInt;
 
 import static com.google.common.base.MoreObjects.toStringHelper;
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.Maps.transformValues;
 import static java.util.Objects.requireNonNull;
 
 public class Footer
 {
     private final long numberOfRows;
-    private final int rowsInRowGroup;
+    private final OptionalInt rowsInRowGroup;
     private final List<StripeInformation> stripes;
     private final ColumnMetadata<OrcType> types;
     private final Optional<ColumnMetadata<ColumnStatistics>> fileStats;
@@ -38,13 +40,14 @@ public class Footer
 
     public Footer(
             long numberOfRows,
-            int rowsInRowGroup,
+            OptionalInt rowsInRowGroup,
             List<StripeInformation> stripes,
             ColumnMetadata<OrcType> types,
             Optional<ColumnMetadata<ColumnStatistics>> fileStats,
             Map<String, Slice> userMetadata)
     {
         this.numberOfRows = numberOfRows;
+        rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1"));
         this.rowsInRowGroup = rowsInRowGroup;
         this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null"));
         this.types = requireNonNull(types, "types is null");
@@ -58,7 +61,7 @@ public long getNumberOfRows()
         return numberOfRows;
     }
 
-    public int getRowsInRowGroup()
+    public OptionalInt getRowsInRowGroup()
     {
         return rowsInRowGroup;
     }
diff --git a/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataReader.java b/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataReader.java
index df9d5b7b6234..5b7c05808b49 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataReader.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataReader.java
@@ -46,6 +46,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.TimeZone;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -134,7 +135,7 @@ public Footer readFooter(HiveWriterVersion hiveWriterVersion, InputStream inputS
         OrcProto.Footer footer = OrcProto.Footer.parseFrom(input);
         return new Footer(
                 footer.getNumberOfRows(),
-                footer.getRowIndexStride(),
+                footer.getRowIndexStride() == 0 ? OptionalInt.empty() : OptionalInt.of(footer.getRowIndexStride()),
                 toStripeInformation(footer.getStripesList()),
                 toType(footer.getTypesList()),
                 toColumnStatistics(hiveWriterVersion, footer.getStatisticsList(), false),
diff --git a/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataWriter.java b/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataWriter.java
index 56f28f399a5d..1e432ea8f157 100644
--- a/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataWriter.java
+++ b/presto-orc/src/main/java/io/prestosql/orc/metadata/OrcMetadataWriter.java
@@ -114,7 +114,7 @@ public int writeFooter(SliceOutput output, Footer footer)
     {
         OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder()
                 .setNumberOfRows(footer.getNumberOfRows())
-                .setRowIndexStride(footer.getRowsInRowGroup())
+                .setRowIndexStride(footer.getRowsInRowGroup().orElse(0))
                 .addAllStripes(footer.getStripes().stream()
                         .map(OrcMetadataWriter::toStripeInformation)
                         .collect(toList()))


Test patch:
diff --git a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java
index 0d292f96d819..dcff9619eb95 100644
--- a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java
+++ b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java
@@ -573,7 +573,7 @@ static OrcRecordReader createCustomOrcRecordReader(TempFile tempFile, OrcPredica
         OrcReader orcReader = new OrcReader(orcDataSource, READER_OPTIONS);
 
         assertEquals(orcReader.getColumnNames(), ImmutableList.of("test"));
-        assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000);
+        assertEquals(orcReader.getFooter().getRowsInRowGroup().orElse(0), 10_000);
 
         return orcReader.createRecordReader(
                 orcReader.getRootColumn().getNestedColumns(),
diff --git a/presto-orc/src/test/java/io/prestosql/orc/TestReadBloomFilter.java b/presto-orc/src/test/java/io/prestosql/orc/TestReadBloomFilter.java
index 5791d73a76bb..8bd5c70e1c03 100644
--- a/presto-orc/src/test/java/io/prestosql/orc/TestReadBloomFilter.java
+++ b/presto-orc/src/test/java/io/prestosql/orc/TestReadBloomFilter.java
@@ -133,7 +133,7 @@ private static OrcRecordReader createCustomOrcRecordReader(TempFile tempFile, Or
         OrcReader orcReader = new OrcReader(orcDataSource, READER_OPTIONS);
 
         assertEquals(orcReader.getColumnNames(), ImmutableList.of("test"));
-        assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000);
+        assertEquals(orcReader.getFooter().getRowsInRowGroup().orElse(0), 10_000);
 
         return orcReader.createRecordReader(
                 orcReader.getRootColumn().getNestedColumns(),
diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveTransactionalTable.java b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveTransactionalTable.java
index 94fbe3907507..49e9e7ff40ec 100644
--- a/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveTransactionalTable.java
+++ b/presto-product-tests/src/main/java/io/prestosql/tests/hive/TestHiveTransactionalTable.java
@@ -17,6 +17,8 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
 import java.util.stream.Stream;
 
 import static io.prestosql.tempto.assertions.QueryAssert.Row.row;
@@ -32,8 +34,9 @@
 public class TestHiveTransactionalTable
         extends HiveProductTest
 {
-    @Test(groups = {STORAGE_FORMATS, HIVE_TRANSACTIONAL}, dataProvider = "partitioningAndBucketingTypeDataProvider")
+    @Test(groups = {STORAGE_FORMATS, HIVE_TRANSACTIONAL}, dataProvider = "partitioningAndBucketingTypeDataProvider", timeOut = 5 * 60 * 1000)
     public void testReadFullAcid(boolean isPartitioned, BucketingType bucketingType)
+            throws InterruptedException, ExecutionException, TimeoutException
     {
         if (getHiveVersionMajor() < 3) {
             throw new SkipException("Presto Hive transactional tables are supported with Hive version 3 or above");
@@ -59,21 +62,30 @@ public void testReadFullAcid(boolean isPartitioned, BucketingType bucketingType)
             // test filtering
             assertThat(query("SELECT col, fcol FROM " + tableName + " WHERE fcol = 1 ORDER BY col")).containsOnly(row(21, 1));
 
+            // test minor compacted data read
+            onHive().executeQuery("INSERT INTO TABLE " + tableName + hivePartitionString + " VALUES (20, 3)");
+            onHive().executeQuery("ALTER TABLE " + tableName + " " + hivePartitionString + " COMPACT 'MINOR' AND WAIT");
+            assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(20, 3), row(21, 1), row(22, 2));
+
             // delete a row
             onHive().executeQuery("DELETE FROM " + tableName + " WHERE fcol=2");
-            assertThat(query(selectFromOnePartitionsSql)).containsOnly(row(21, 1));
+            assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(20, 3), row(21, 1));
 
             // update the existing row
             String predicate = "fcol = 1" + (isPartitioned ? " AND part_col = 2 " : "");
             onHive().executeQuery("UPDATE " + tableName + " SET col = 23 WHERE " + predicate);
-            assertThat(query(selectFromOnePartitionsSql)).containsOnly(row(23, 1));
+            assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(20, 3), row(23, 1));
+
+            // test major compaction
+            onHive().executeQuery("ALTER TABLE " + tableName + " " + hivePartitionString + " COMPACT 'MAJOR' AND WAIT");
+            assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(20, 3), row(23, 1));
         }
         finally {
             onHive().executeQuery("DROP TABLE " + tableName);
         }
     }
 
-    @Test(groups = {STORAGE_FORMATS, HIVE_TRANSACTIONAL}, dataProvider = "partitioningAndBucketingTypeDataProvider")
+    @Test(groups = {STORAGE_FORMATS, HIVE_TRANSACTIONAL}, dataProvider = "partitioningAndBucketingTypeDataProvider", timeOut = 5 * 60 * 1000)
     public void testReadInsertOnly(boolean isPartitioned, BucketingType bucketingType)
     {
         if (getHiveVersionMajor() < 3) {
@@ -98,8 +110,17 @@ public void testReadInsertOnly(boolean isPartitioned, BucketingType bucketingTyp
             onHive().executeQuery("INSERT INTO TABLE " + tableName + hivePartitionString + " SELECT 2");
             assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(1), row(2));
 
+            // test minor compacted data read
+            onHive().executeQuery("ALTER TABLE " + tableName + " " + hivePartitionString + " COMPACT 'MINOR' AND WAIT");
+            assertThat(query(selectFromOnePartitionsSql)).containsExactly(row(1), row(2));
+
             onHive().executeQuery("INSERT OVERWRITE TABLE " + tableName + hivePartitionString + " SELECT 3");
             assertThat(query(selectFromOnePartitionsSql)).containsOnly(row(3));
+
+            // test major compaction
+            onHive().executeQuery("INSERT INTO TABLE " + tableName + hivePartitionString + " SELECT 4");
+            onHive().executeQuery("ALTER TABLE " + tableName + " " + hivePartitionString + " COMPACT 'MAJOR' AND WAIT");
+            assertThat(query(selectFromOnePartitionsSql)).containsOnly(row(3), row(4));
         }
         finally {
             onHive().executeQuery("DROP TABLE " + tableName);