ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35899][SQL][FOLLOWUP] Utility to convert connector expressions to Catalyst

Browse source 
### What changes were proposed in this pull request?

This PR addresses post-review comments on PR #33096:
- removes `private[sql]` modifier
- removes the option to pass a resolver to simplify the API

### Why are the changes needed?

These changes are needed to simply the utility API.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #33120 from aokolnychyi/spark-35899-follow-up.

Authored-by: Anton Okolnychyi <aokolnychyi@apple.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Anton Okolnychyi 2021-06-28 22:22:07 -07:00 committed by Dongjoon Hyun
parent c45a6f5d09
commit 8a21d2dcfe
2 changed files with 12 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
View file

@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.SQLConfHelper
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.connector.expressions.{Expression => V2Expression, FieldReference, IdentityTransform, NamedReference, NullOrdering => V2NullOrdering, SortDirection => V2SortDirection, SortValue}
import org.apache.spark.sql.errors.QueryCompilationErrors
@ -27,14 +26,11 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
/**
* A utility class that converts public connector expressions into Catalyst expressions.
*/
private[sql] object V2ExpressionUtils extends SQLConfHelper {
object V2ExpressionUtils extends SQLConfHelper {
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
def resolveRef[T <: NamedExpression](
ref: NamedReference,
plan: LogicalPlan,
resolver: Resolver = conf.resolver): T = {
plan.resolve(ref.fieldNames, resolver) match {
def resolveRef[T <: NamedExpression](ref: NamedReference, plan: LogicalPlan): T = {
plan.resolve(ref.fieldNames, conf.resolver) match {
case Some(namedExpr) =>
namedExpr.asInstanceOf[T]
case None =>
@ -44,25 +40,19 @@ private[sql] object V2ExpressionUtils extends SQLConfHelper {
}
}
def resolveRefs[T <: NamedExpression](
refs: Seq[NamedReference],
plan: LogicalPlan,
resolver: Resolver = conf.resolver): Seq[T] = {
refs.map(ref => resolveRef[T](ref, plan, resolver))
def resolveRefs[T <: NamedExpression](refs: Seq[NamedReference], plan: LogicalPlan): Seq[T] = {
refs.map(ref => resolveRef[T](ref, plan))
}
def toCatalyst(
expr: V2Expression,
query: LogicalPlan,
resolver: Resolver = conf.resolver): Expression = {
def toCatalyst(expr: V2Expression, query: LogicalPlan): Expression = {
expr match {
case SortValue(child, direction, nullOrdering) =>
val catalystChild = toCatalyst(child, query, resolver)
val catalystChild = toCatalyst(child, query)
SortOrder(catalystChild, toCatalyst(direction), toCatalyst(nullOrdering), Seq.empty)
case IdentityTransform(ref) =>
resolveRef[NamedExpression](ref, query, resolver)
resolveRef[NamedExpression](ref, query)
case ref: FieldReference =>
resolveRef[NamedExpression](ref, query, resolver)
resolveRef[NamedExpression](ref, query)
case _ =>
throw new AnalysisException(s"$expr is not currently supported")
}

8
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DistributionAndOrderingUtils.scala
View file

@ -29,12 +29,10 @@ object DistributionAndOrderingUtils {
def prepareQuery(write: Write, query: LogicalPlan, conf: SQLConf): LogicalPlan = write match {
case write: RequiresDistributionAndOrdering =>
val resolver = conf.resolver
val numPartitions = write.requiredNumPartitions()
val distribution = write.requiredDistribution match {
case d: OrderedDistribution => d.ordering.map(e => toCatalyst(e, query, resolver))
case d: ClusteredDistribution => d.clustering.map(e => toCatalyst(e, query, resolver))
case d: OrderedDistribution => d.ordering.map(e => toCatalyst(e, query))
case d: ClusteredDistribution => d.clustering.map(e => toCatalyst(e, query))
case _: UnspecifiedDistribution => Array.empty[Expression]
}
@ -55,7 +53,7 @@ object DistributionAndOrderingUtils {
}
val ordering = write.requiredOrdering.toSeq
.map(e => toCatalyst(e, query, resolver))
.map(e => toCatalyst(e, query))
.asInstanceOf[Seq[SortOrder]]
val queryWithDistributionAndOrdering = if (ordering.nonEmpty) {