Repository: trinodb/trino (Java)
Date: 2020-08-28 13:56:30+00:00
Task type: Bug Fix

Issue:
PrestoS3FileSystem should skip DeepArchive in addition to Glacier
The `skip-glacier-objects` config is intended to skip all Glacier objects and thus should apply to both storage classes.


Issue discussion:

Solution patch:
diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java
index 7789085c2daa..dec7e87b9a3f 100644
--- a/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java
+++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/s3/PrestoS3FileSystem.java
@@ -61,6 +61,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.collect.AbstractSequentialIterator;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterators;
 import com.google.common.io.Closer;
 import io.airlift.log.Logger;
@@ -102,11 +103,13 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.amazonaws.regions.Regions.US_EAST_1;
 import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION;
 import static com.amazonaws.services.s3.Headers.UNENCRYPTED_CONTENT_LENGTH;
+import static com.amazonaws.services.s3.model.StorageClass.DeepArchive;
 import static com.amazonaws.services.s3.model.StorageClass.Glacier;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkPositionIndexes;
@@ -182,6 +185,7 @@ public class PrestoS3FileSystem
     private static final Duration BACKOFF_MIN_SLEEP = new Duration(1, SECONDS);
     private static final int HTTP_RANGE_NOT_SATISFIABLE = 416;
     private static final String S3_CUSTOM_SIGNER = "PrestoS3CustomSigner";
+    private static final Set<String> GLACIER_STORAGE_CLASSES = ImmutableSet.of(Glacier.toString(), DeepArchive.toString());
 
     private URI uri;
     private Path workingDirectory;
@@ -634,7 +638,7 @@ private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> obje
 
     private static boolean isGlacierObject(S3ObjectSummary object)
     {
-        return Glacier.toString().equals(object.getStorageClass());
+        return GLACIER_STORAGE_CLASSES.contains(object.getStorageClass());
     }
 
     private static boolean isHadoopFolderMarker(S3ObjectSummary object)


Test patch:
diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/MockAmazonS3.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/MockAmazonS3.java
index 7465e5e0f25d..7fc04342ace4 100644
--- a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/MockAmazonS3.java
+++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/MockAmazonS3.java
@@ -121,6 +121,12 @@ public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Reque
                 glacier.setKey("test/glacier");
                 glacier.setLastModified(new Date());
                 listingV2.getObjectSummaries().add(glacier);
+
+                S3ObjectSummary deepArchive = new S3ObjectSummary();
+                deepArchive.setStorageClass(StorageClass.DeepArchive.toString());
+                deepArchive.setKey("test/deepArchive");
+                deepArchive.setLastModified(new Date());
+                listingV2.getObjectSummaries().add(deepArchive);
             }
         }
         else {
diff --git a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestPrestoS3FileSystem.java b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestPrestoS3FileSystem.java
index d61a0d0bcd80..6fe4a7c957eb 100644
--- a/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestPrestoS3FileSystem.java
+++ b/presto-hive/src/test/java/io/prestosql/plugin/hive/s3/TestPrestoS3FileSystem.java
@@ -577,7 +577,7 @@ private static void assertSkipGlacierObjects(boolean skipGlacierObjects)
             fs.initialize(new URI("s3n://test-bucket/"), config);
             fs.setS3Client(s3);
             FileStatus[] statuses = fs.listStatus(new Path("s3n://test-bucket/test"));
-            assertEquals(statuses.length, skipGlacierObjects ? 2 : 3);
+            assertEquals(statuses.length, skipGlacierObjects ? 2 : 4);
         }
     }