[SPARK-36130][SQL] UnwrapCastInBinaryComparison should skip In expression when in.list contains an expression that is not literal
### What changes were proposed in this pull request? Fix [comment](https://github.com/apache/spark/pull/32488#issuecomment-879315179) This PR fix rule `UnwrapCastInBinaryComparison` bug. Rule UnwrapCastInBinaryComparison should skip In expression when in.list contains an expression that is not literal. - In Before this pr, the following example will throw an exception. ```scala withTable("tbl") { sql("CREATE TABLE tbl (d decimal(33, 27)) USING PARQUET") sql("SELECT d FROM tbl WHERE d NOT IN (d + 1)") } ``` - InSet As the analyzer guarantee that all the elements in the `inSet.hset` are literal, so this is not an issue for `InSet`.fbf53dee37/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala (L264-L279)
### Does this PR introduce _any_ user-facing change? No, only bug fix. ### How was this patch tested? New test. Closes #33335 from cfmcgrady/SPARK-36130. Authored-by: Fu Chen <cfmcgrady@gmail.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com> (cherry picked from commit103d16e868
) Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
78796349d9
commit
5bc06fd7d9
|
@ -141,8 +141,9 @@ object UnwrapCastInBinaryComparison extends Rule[LogicalPlan] {
|
|||
// values.
|
||||
// 2. this rule only handles the case when both `fromExp` and value in `in.list` are of numeric
|
||||
// type.
|
||||
// 3. this rule doesn't optimize In when `in.list` contains an expression that is not literal.
|
||||
case in @ In(Cast(fromExp, toType: NumericType, _, _), list @ Seq(firstLit, _*))
|
||||
if canImplicitlyCast(fromExp, toType, firstLit.dataType) =>
|
||||
if canImplicitlyCast(fromExp, toType, firstLit.dataType) && in.inSetConvertible =>
|
||||
|
||||
// There are 3 kinds of literals in the list:
|
||||
// 1. null literals
|
||||
|
|
|
@ -283,6 +283,25 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with ExpressionEvalHelp
|
|||
)
|
||||
}
|
||||
|
||||
test("SPARK-36130: unwrap In should skip when in.list contains an expression that " +
|
||||
"is not literal") {
|
||||
val add = Cast(f2, DoubleType) + 1.0d
|
||||
val doubleLit = Literal.create(null, DoubleType)
|
||||
assertEquivalent(In(Cast(f2, DoubleType), Seq(add)), In(Cast(f2, DoubleType), Seq(add)))
|
||||
assertEquivalent(
|
||||
In(Cast(f2, DoubleType), Seq(doubleLit, add)),
|
||||
In(Cast(f2, DoubleType), Seq(doubleLit, add)))
|
||||
assertEquivalent(
|
||||
In(Cast(f2, DoubleType), Seq(doubleLit, 1.0d, add)),
|
||||
In(Cast(f2, DoubleType), Seq(doubleLit, 1.0d, add)))
|
||||
assertEquivalent(
|
||||
In(Cast(f2, DoubleType), Seq(1.0d, add)),
|
||||
In(Cast(f2, DoubleType), Seq(1.0d, add)))
|
||||
assertEquivalent(
|
||||
In(Cast(f2, DoubleType), Seq(0.0d, 1.0d, add)),
|
||||
In(Cast(f2, DoubleType), Seq(0.0d, 1.0d, add)))
|
||||
}
|
||||
|
||||
private def castInt(e: Expression): Expression = Cast(e, IntegerType)
|
||||
private def castDouble(e: Expression): Expression = Cast(e, DoubleType)
|
||||
private def castDecimal2(e: Expression): Expression = Cast(e, DecimalType(10, 4))
|
||||
|
|
Loading…
Reference in a new issue