Project: talenddataquality
Issue TDQ-11881: Spark doesn't have permission to delete the window file
I have a Big Data job which used tStandardizeRow. In my Spark Configuration, I used "Use local mode", spark version "1.6.0", Spark scratch directory "/DQ". After run the job, I have my expected correct results, but have also these errors below: [ERROR]: org.apache.spark.util.ShutdownHookManager - Exception while deleting Spark temp dir: C:\DQ\spark-5c0df763-2e9c-4747-af46-4d9f6051f77b java.io.IOException: Failed to delete: C:\DQ\spark-5c0df763-2e9c-4747-af46-4d9f6051f77b org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:928) org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:65) org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:62) scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108) org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:62) org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:267) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1741) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239) scala.util.Try$.apply(Try.scala:161) org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:218) org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) [ERROR]: org.apache.spark.util.ShutdownHookManager - Exception while deleting Spark temp dir: C:\DQ\spark-5c0df763-2e9c-4747-af46-4d9f6051f77b\userFiles-c6d6fa0a-76a3-43dc-98f6-814fbf1b96d1 java.io.IOException: Failed to delete: C:\DQ\spark-5c0df763-2e9c-4747-af46-4d9f6051f77b\userFiles-c6d6fa0a-76a3-43dc-98f6-814fbf1b96d1 org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:928) org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:65) org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:62) scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108) org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:62) org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:267) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1741) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:239) scala.util.Try$.apply(Try.scala:161) org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:239) org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:218) org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) Job Copy_of_test11489 ended at 18:49 12/04/2016. [exit code=0]
Story points: 3