diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala index eb8022c8c8..7f49d8db48 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala @@ -30,16 +30,21 @@ import org.apache.spark.sql.types._ */ object NestedColumnAliasing { - def unapply(plan: LogicalPlan) - : Option[(Map[ExtractValue, Alias], Map[ExprId, Seq[Alias]])] = plan match { + def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match { case Project(projectList, child) if SQLConf.get.nestedSchemaPruningEnabled && canProjectPushThrough(child) => val exprCandidatesToPrune = projectList ++ child.expressions - getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq) + getAliasSubMap(exprCandidatesToPrune, child.producedAttributes.toSeq).map { + case (nestedFieldToAlias, attrToAliases) => + NestedColumnAliasing.replaceToAliases(plan, nestedFieldToAlias, attrToAliases) + } - case plan if SQLConf.get.nestedSchemaPruningEnabled && canPruneOn(plan) => - val exprCandidatesToPrune = plan.expressions - getAliasSubMap(exprCandidatesToPrune, plan.producedAttributes.toSeq) + case p if SQLConf.get.nestedSchemaPruningEnabled && canPruneOn(p) => + val exprCandidatesToPrune = p.expressions + getAliasSubMap(exprCandidatesToPrune, p.producedAttributes.toSeq).map { + case (nestedFieldToAlias, attrToAliases) => + NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases) + } case _ => None } @@ -47,7 +52,7 @@ object NestedColumnAliasing { /** * Replace nested columns to prune unused nested columns later. */ - def replaceToAliases( + private def replaceToAliases( plan: LogicalPlan, nestedFieldToAlias: Map[ExtractValue, Alias], attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = plan match { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index a1a7213664..8e57e9737c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -649,8 +649,7 @@ object ColumnPruning extends Rule[LogicalPlan] { // Can't prune the columns on LeafNode case p @ Project(_, _: LeafNode) => p - case p @ NestedColumnAliasing(nestedFieldToAlias, attrToAliases) => - NestedColumnAliasing.replaceToAliases(p, nestedFieldToAlias, attrToAliases) + case NestedColumnAliasing(p) => p // for all other logical plans that inherits the output from it's children // Project over project is handled by the first case, skip it here.