[SPARK-34595][SQL] DPP support RLIKE
### What changes were proposed in this pull request? This pr make DPP support RLIKE expression: ```sql SELECT date_id, product_id FROM fact_sk f JOIN dim_store s ON f.store_id = s.store_id WHERE s.country RLIKE '[DE|US]' ``` ### Why are the changes needed? Improve query performance. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit test. Closes #31722 from chaojun-zhang/SPARK-34595. Authored-by: helloman <zcj23085@gmail.com> Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
parent
f72b9068ad
commit
1a9722420e
|
@ -160,7 +160,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
|
|||
case Not(expr) => isLikelySelective(expr)
|
||||
case And(l, r) => isLikelySelective(l) || isLikelySelective(r)
|
||||
case Or(l, r) => isLikelySelective(l) && isLikelySelective(r)
|
||||
case Like(_, _, _) => true
|
||||
case _: StringRegexExpression => true
|
||||
case _: BinaryComparison => true
|
||||
case _: In | _: InSet => true
|
||||
case _: StringPredicate => true
|
||||
|
|
|
@ -1403,6 +1403,32 @@ abstract class DynamicPartitionPruningSuiteBase
|
|||
)
|
||||
}
|
||||
}
|
||||
|
||||
test("SPARK-34595: DPP support RLIKE expression") {
|
||||
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
|
||||
val df = sql(
|
||||
"""
|
||||
|SELECT date_id, product_id FROM fact_sk f
|
||||
|JOIN dim_store s
|
||||
|ON f.store_id = s.store_id WHERE s.country RLIKE '[DE|US]'
|
||||
""".stripMargin)
|
||||
|
||||
checkPartitionPruningPredicate(df, false, true)
|
||||
|
||||
checkAnswer(df,
|
||||
Row(1030, 2) ::
|
||||
Row(1040, 2) ::
|
||||
Row(1050, 2) ::
|
||||
Row(1060, 2) ::
|
||||
Row(1070, 2) ::
|
||||
Row(1080, 3) ::
|
||||
Row(1090, 3) ::
|
||||
Row(1100, 3) ::
|
||||
Row(1110, 3) ::
|
||||
Row(1120, 4) :: Nil
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase
|
||||
|
|
Loading…
Reference in a new issue