amplab/shark — pull request #212 by gao8658 (2013-11-06 14:02:58+00:00)
Title: Branch 0.8-In MapSplitPruning.scala  when execute sql has  between operator  will cause  java.lang.ClassCastException

I excute the query

select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from
store_sales_cached ss join date_dim_cached dd on(ss.ss_sold_date_sk = dd.d_date_sk)
where d_month_seq between 1176 and 1176+11 group by ss_store_sk, ss_item_sk limit 1;
~
~

The exception let shark-0.8.0 shut down

java.lang.ClassCastException: org.apache.hadoop.hive.ql.exec.ExprNodeGenericFuncEvaluator cannot be cast to org.apache.hadoop.hive.ql.exec.ExprNodeConstantEvaluator
at org.apache.hadoop.hive.ql.exec.MapSplitPruning$.test(MapSplitPruning.scala:66)
at shark.execution.TableScanOperator.prunePartitionFunc$1(TableScanOperator.scala:186)
at shark.execution.TableScanOperator$$anonfun$1.apply$mcZI$sp(TableScanOperator.scala:193)
at org.apache.spark.rdd.PruneDependency$$anonfun$1.apply(PartitionPruningRDD.scala:37)
at org.apache.spark.rdd.PruneDependency$$anonfun$1.apply(PartitionPruningRDD.scala:37)
at scala.collection.TraversableLike$$anonfun$filter$1.apply(TraversableLike.scala:252)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:34)
at scala.collection.mutable.ArrayOps.foreach(ArrayOps.scala:38)
at scala.collection.TraversableLike$class.filter(TraversableLike.scala:251)
at scala.collection.mutable.ArrayOps.filter(ArrayOps.scala:38)
at org.apache.spark.rdd.PruneDependency.(PartitionPruningRDD.scala:37)
at org.apache.spark.rdd.PartitionPruningRDD.(PartitionPruningRDD.scala:53)
at org.apache.spark.rdd.PartitionPruningRDD$.create(PartitionPruningRDD.scala:70)
at shark.execution.TableScanOperator.createPrunedRdd(TableScanOperator.scala:193)
at shark.execution.TableScanOperator.execute(TableScanOperator.scala:130)
at shark.execution.Operator$$anonfun$executeParents$1.apply(Operator.scala:115)
at shark.execution.Operator$$anonfun$executeParents$1.apply(Operator.scala:115)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:233)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:233)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:60)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:233)
at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:47)
at shark.execution.Operator.executeParents(Operator.scala:115)
at shark.execution.UnaryOperator.execute(Operator.scala:187)
at shark.execution.Operator$$anonfun$executeParents$1.apply(Operator.scala:115)

and the code is in MapSplitPruning.scala

def test(s: TablePartitionStats, e: ExprNodeEvaluator): Boolean = {
if (s.numRows == 0) {
// If the partition is empty, it can be pruned.
false
} else {
// If the partition is not empty, test the condition based on the operator.
e match {
case e: ExprNodeGenericFuncEvaluator => {
e.genericUDF match {
case _: GenericUDFOPAnd => test(s, e.children(0)) && test(s, e.children(1))
case _: GenericUDFOPOr => test(s, e.children(0)) || test(s, e.children(1))
case _: GenericUDFBetween =>
val col = e.children(1)
if (col.isInstanceOf[ExprNodeColumnEvaluator]) {
testBetweenPredicate(s, e.children(0).asInstanceOf[ExprNodeConstantEvaluator],
col.asInstanceOf[ExprNodeColumnEvaluator],
e.children(2).asInstanceOf[ExprNodeConstantEvaluator],
e.children(3).asInstanceOf[ExprNodeConstantEvaluator])
} else {
//cannot prune function based evaluators in general.
true
}

case _: GenericUDFIn =>
testInPredicate(s, e.children(0).asInstanceOf[ExprNodeColumnEvaluator], e.children.drop(1))
case udf: GenericUDFBaseCompare =>
testComparisonPredicate(s, udf, e.children(0), e.children(1))
case _ => true
}
}
case _ => true
}
}
}

I think there are type cast error in

testBetweenPredicate(s, e.children(0).asInstanceOf[ExprNodeConstantEvaluator],
col.asInstanceOf[ExprNodeColumnEvaluator],
e.children(2).asInstanceOf[ExprNodeConstantEvaluator],
e.children(3).asInstanceOf[ExprNodeConstantEvaluator])