[SPARK-31977][SQL] Returns the plan directly from NestedColumnAliasing
### What changes were proposed in this pull request? This proposes a minor refactoring to match `NestedColumnAliasing` to `GeneratorNestedColumnAliasing` so it returns the pruned plan directly. ```scala case p NestedColumnAliasing(nestedFieldToAlias, attrToAliases) => NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases) ``` vs ```scala case GeneratorNestedColumnAliasing(p) => p ``` ### Why are the changes needed? Just for readability. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests should cover. Closes #28812 from HyukjinKwon/SPARK-31977. Authored-by: HyukjinKwon <gurwls223@apache.org> Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
parent
78d08a8c38
commit
a620a2a7e5
|
@ -30,16 +30,21 @@ import org.apache.spark.sql.types._
|
||||||
*/
|
*/
|
||||||
object NestedColumnAliasing {
|
object NestedColumnAliasing {
|
||||||
|
|
||||||
def unapply(plan: LogicalPlan)
|
def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
|
||||||
: Option[(Map[ExtractValue, Alias], Map[ExprId, Seq[Alias]])] = plan match {
|
|
||||||
case Project(projectList, child)
|
case Project(projectList, child)
|
||||||
if SQLConf.get.nestedSchemaPruningEnabled && canProjectPushThrough(child) =>
|
if SQLConf.get.nestedSchemaPruningEnabled && canProjectPushThrough(child) =>
|
||||||
val exprCandidatesToPrune = projectList ++ child.expressions
|
val exprCandidatesToPrune = projectList ++ child.expressions
|
||||||
getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq)
|
getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq).map {
|
||||||
|
case (nestedFieldToAlias, attrToAliases) =>
|
||||||
|
NestedColumnAliasing.replaceToAliases(plan, nestedFieldToAlias, attrToAliases)
|
||||||
|
}
|
||||||
|
|
||||||
case plan if SQLConf.get.nestedSchemaPruningEnabled && canPruneOn(plan) =>
|
case p if SQLConf.get.nestedSchemaPruningEnabled && canPruneOn(p) =>
|
||||||
val exprCandidatesToPrune = plan.expressions
|
val exprCandidatesToPrune = p.expressions
|
||||||
getAliasSubMap(exprCandidatesToPrune, plan.producedAttributes.toSeq)
|
getAliasSubMap(exprCandidatesToPrune, p.producedAttributes.toSeq).map {
|
||||||
|
case (nestedFieldToAlias, attrToAliases) =>
|
||||||
|
NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases)
|
||||||
|
}
|
||||||
|
|
||||||
case _ => None
|
case _ => None
|
||||||
}
|
}
|
||||||
|
@ -47,7 +52,7 @@ object NestedColumnAliasing {
|
||||||
/**
|
/**
|
||||||
* Replace nested columns to prune unused nested columns later.
|
* Replace nested columns to prune unused nested columns later.
|
||||||
*/
|
*/
|
||||||
def replaceToAliases(
|
private def replaceToAliases(
|
||||||
plan: LogicalPlan,
|
plan: LogicalPlan,
|
||||||
nestedFieldToAlias: Map[ExtractValue, Alias],
|
nestedFieldToAlias: Map[ExtractValue, Alias],
|
||||||
attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = plan match {
|
attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = plan match {
|
||||||
|
|
|
@ -649,8 +649,7 @@ object ColumnPruning extends Rule[LogicalPlan] {
|
||||||
// Can't prune the columns on LeafNode
|
// Can't prune the columns on LeafNode
|
||||||
case p @ Project(_, _: LeafNode) => p
|
case p @ Project(_, _: LeafNode) => p
|
||||||
|
|
||||||
case p @ NestedColumnAliasing(nestedFieldToAlias, attrToAliases) =>
|
case NestedColumnAliasing(p) => p
|
||||||
NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases)
|
|
||||||
|
|
||||||
// for all other logical plans that inherits the output from it's children
|
// for all other logical plans that inherits the output from it's children
|
||||||
// Project over project is handled by the first case, skip it here.
|
// Project over project is handled by the first case, skip it here.
|
||||||
|
|
Loading…
Reference in a new issue