ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-34595][SQL] DPP support RLIKE

Browse source 
### What changes were proposed in this pull request?
This pr make DPP support RLIKE expression:

```sql
SELECT date_id, product_id FROM fact_sk f
JOIN dim_store s
ON f.store_id = s.store_id WHERE s.country RLIKE  '[DE|US]'
```
 ### Why are the changes needed?
Improve query performance.

 ### Does this PR introduce _any_ user-facing change?
 No.

 ### How was this patch tested?
 Unit test.

Closes #31722 from chaojun-zhang/SPARK-34595.

Authored-by: helloman <zcj23085@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
helloman 2021-03-06 20:30:20 +09:00 committed by Takeshi Yamamuro
parent f72b9068ad
commit 1a9722420e
2 changed files with 27 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
View file

@ -160,7 +160,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
case Not(expr) => isLikelySelective(expr) case Not(expr) => isLikelySelective(expr)
case And(l, r) => isLikelySelective(l) || isLikelySelective(r) case And(l, r) => isLikelySelective(l) || isLikelySelective(r)
case Or(l, r) => isLikelySelective(l) && isLikelySelective(r) case Or(l, r) => isLikelySelective(l) && isLikelySelective(r)
case Like(_, _, _) => true case _: StringRegexExpression => true
case _: BinaryComparison => true case _: BinaryComparison => true
case _: In | _: InSet => true case _: In | _: InSet => true
case _: StringPredicate => true case _: StringPredicate => true

26
sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
View file

@ -1403,6 +1403,32 @@ abstract class DynamicPartitionPruningSuiteBase
) )
} }
} }
test("SPARK-34595: DPP support RLIKE expression") {
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
val df = sql(
"""
|SELECT date_id, product_id FROM fact_sk f
|JOIN dim_store s
|ON f.store_id = s.store_id WHERE s.country RLIKE '[DE|US]'
""".stripMargin)
checkPartitionPruningPredicate(df, false, true)
checkAnswer(df,
Row(1030, 2) ::
Row(1040, 2) ::
Row(1050, 2) ::
Row(1060, 2) ::
Row(1070, 2) ::
Row(1080, 3) ::
Row(1090, 3) ::
Row(1100, 3) ::
Row(1110, 3) ::
Row(1120, 4) :: Nil
)
}
}
} }
class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase