Bug: insert overwrite failed for external table with dynamic partition table
Status: Open | Priority: Major | Resolution: 
Created: 15/Feb/22 10:02
Affects Version/s: 3.2.1

can't insert overwrite dynamic partition table, reproduce step with spark3.2.1 hadoop 3.2:

sql("CREATE EXTERNAL TABLE exttb01(id int) PARTITIONED BY (p1 string, p2 string) STORED AS PARQUET LOCATION '/tmp/exttb01'")
sql("set spark.sql.hive.convertMetastoreParquet=false")
sql("set hive.exec.dynamic.partition.mode=nonstrict")
val insertsql = "INSERT OVERWRITE TABLE exttb01 PARTITION(p1='n1', p2) SELECT * FROM VALUES (1, 'n2'), (2, 'n3'), (3, 'n4') AS t(id, p2)"
sql(insertsql)
sql(insertsql)

when execute insert overwrite 2th time, it failed

 

WARN Hive: Directory file:/tmp/exttb01/p1=n1/p2=n4 cannot be cleaned: java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n4 does not exist
java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n4 does not exist
        at org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:597)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:761)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.hive.ql.metadata.Hive.replaceFiles(Hive.java:3440)
        at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1657)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1929)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1920)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
22/02/15 17:59:19 WARN Hive: Directory file:/tmp/exttb01/p1=n1/p2=n3 cannot be cleaned: java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n3 does not exist
java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n3 does not exist
        at org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:597)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:761)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.hive.ql.metadata.Hive.replaceFiles(Hive.java:3440)
        at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1657)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1929)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1920)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
22/02/15 17:59:19 WARN Hive: Directory file:/tmp/exttb01/p1=n1/p2=n2 cannot be cleaned: java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n2 does not exist
java.io.FileNotFoundException: File file:/tmp/exttb01/p1=n1/p2=n2 does not exist
        at org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:597)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:761)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972)
        at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014)
        at org.apache.hadoop.hive.ql.metadata.Hive.replaceFiles(Hive.java:3440)
        at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1657)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1929)
        at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1920)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)