[SPARK-33951][SQL] Distinguish the error between filter and distinct
### What changes were proposed in this pull request? The error messages for specifying filter and distinct for the aggregate function are mixed together and should be separated. This can increase readability and ease of use. ### Why are the changes needed? increase readability and ease of use. ### Does this PR introduce _any_ user-facing change? 'Yes'. ### How was this patch tested? Jenkins test Closes #30982 from beliefer/SPARK-33951. Lead-authored-by: gengjiaan <gengjiaan@360.cn> Co-authored-by: beliefer <beliefer@163.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
gengjiaan
Wenchen Fan
parent
67195d0d97
commit
b037930952
|
|
@ -263,13 +263,8 @@ object QueryCompilationErrors {
|
|||
s"its class is $classCanonicalName, which is not a generator.")
|
||||
}
|
||||
|
||||
def distinctOrFilterOnlyWithAggregateFunctionError(prettyName: String): Throwable = {
|
||||
new AnalysisException("DISTINCT or FILTER specified, " +
|
||||
s"but $prettyName is not an aggregate function")
|
||||
}
|
||||
|
||||
def ignoreNullsWithUnsupportedFunctionError(prettyName: String): Throwable = {
|
||||
new AnalysisException(s"Function $prettyName does not support IGNORE NULLS")
|
||||
def functionWithUnsupportedSyntaxError(prettyName: String, syntax: String): Throwable = {
|
||||
new AnalysisException(s"Function $prettyName does not support $syntax")
|
||||
}
|
||||
|
||||
def nonDeterministicFilterInAggregateError(): Throwable = {
|
||||
|
|
|
|||
|
|
@ -2120,24 +2120,30 @@ class Analyzer(override val catalogManager: CatalogManager)
|
|||
// the context of a Window clause. They do not need to be wrapped in an
|
||||
// AggregateExpression.
|
||||
case wf: AggregateWindowFunction =>
|
||||
if (isDistinct || filter.isDefined) {
|
||||
throw QueryCompilationErrors.distinctOrFilterOnlyWithAggregateFunctionError(
|
||||
wf.prettyName)
|
||||
if (isDistinct) {
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
wf.prettyName, "DISTINCT")
|
||||
} else if (filter.isDefined) {
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
wf.prettyName, "FILTER clause")
|
||||
} else if (ignoreNulls) {
|
||||
wf match {
|
||||
case nthValue: NthValue =>
|
||||
nthValue.copy(ignoreNulls = ignoreNulls)
|
||||
case _ =>
|
||||
throw QueryCompilationErrors.ignoreNullsWithUnsupportedFunctionError(
|
||||
wf.prettyName)
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
wf.prettyName, "IGNORE NULLS")
|
||||
}
|
||||
} else {
|
||||
wf
|
||||
}
|
||||
case owf: FrameLessOffsetWindowFunction =>
|
||||
if (isDistinct || filter.isDefined) {
|
||||
throw QueryCompilationErrors.distinctOrFilterOnlyWithAggregateFunctionError(
|
||||
owf.prettyName)
|
||||
if (isDistinct) {
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
owf.prettyName, "DISTINCT")
|
||||
} else if (filter.isDefined) {
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
owf.prettyName, "FILTER clause")
|
||||
} else if (ignoreNulls) {
|
||||
owf match {
|
||||
case lead: Lead =>
|
||||
|
|
@ -2145,8 +2151,8 @@ class Analyzer(override val catalogManager: CatalogManager)
|
|||
case lag: Lag =>
|
||||
lag.copy(ignoreNulls = ignoreNulls)
|
||||
case _ =>
|
||||
throw QueryCompilationErrors.ignoreNullsWithUnsupportedFunctionError(
|
||||
owf.prettyName)
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
owf.prettyName, "IGNORE NULLS")
|
||||
}
|
||||
} else {
|
||||
owf
|
||||
|
|
@ -2161,20 +2167,23 @@ class Analyzer(override val catalogManager: CatalogManager)
|
|||
case first: First => first.copy(ignoreNulls = ignoreNulls)
|
||||
case last: Last => last.copy(ignoreNulls = ignoreNulls)
|
||||
case _ =>
|
||||
throw QueryCompilationErrors.ignoreNullsWithUnsupportedFunctionError(
|
||||
agg.prettyName)
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
agg.prettyName, "IGNORE NULLS")
|
||||
}
|
||||
AggregateExpression(aggFunc, Complete, isDistinct, filter)
|
||||
} else {
|
||||
AggregateExpression(agg, Complete, isDistinct, filter)
|
||||
}
|
||||
// This function is not an aggregate function, just return the resolved one.
|
||||
case other if (isDistinct || filter.isDefined) =>
|
||||
throw QueryCompilationErrors.distinctOrFilterOnlyWithAggregateFunctionError(
|
||||
other.prettyName)
|
||||
case other if (ignoreNulls) =>
|
||||
throw QueryCompilationErrors.ignoreNullsWithUnsupportedFunctionError(
|
||||
other.prettyName)
|
||||
case other if isDistinct =>
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
other.prettyName, "DISTINCT")
|
||||
case other if filter.isDefined =>
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
other.prettyName, "FILTER clause")
|
||||
case other if ignoreNulls =>
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
other.prettyName, "IGNORE NULLS")
|
||||
case e: String2TrimExpression if arguments.size == 2 =>
|
||||
if (trimWarningEnabled.get) {
|
||||
log.warn("Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated." +
|
||||
|
|
|
|||
|
|
@ -41,7 +41,8 @@ case class ResolveHigherOrderFunctions(catalog: SessionCatalog) extends Rule[Log
|
|||
filter.foreach(_.failAnalysis("FILTER predicate specified, " +
|
||||
s"but ${func.prettyName} is not an aggregate function"))
|
||||
if (ignoreNulls) {
|
||||
throw QueryCompilationErrors.ignoreNullsWithUnsupportedFunctionError(func.prettyName)
|
||||
throw QueryCompilationErrors.functionWithUnsupportedSyntaxError(
|
||||
func.prettyName, "IGNORE NULLS")
|
||||
}
|
||||
func
|
||||
case other => other.failAnalysis(
|
||||
|
|
|
|||
|
|
@ -184,22 +184,22 @@ class AnalysisErrorSuite extends AnalysisTest {
|
|||
errorTest(
|
||||
"distinct function",
|
||||
CatalystSqlParser.parsePlan("SELECT hex(DISTINCT a) FROM TaBlE"),
|
||||
"DISTINCT or FILTER specified, but hex is not an aggregate function" :: Nil)
|
||||
"Function hex does not support DISTINCT" :: Nil)
|
||||
|
||||
errorTest(
|
||||
"non aggregate function with filter predicate",
|
||||
CatalystSqlParser.parsePlan("SELECT hex(a) FILTER (WHERE c = 1) FROM TaBlE2"),
|
||||
"DISTINCT or FILTER specified, but hex is not an aggregate function" :: Nil)
|
||||
"Function hex does not support FILTER clause" :: Nil)
|
||||
|
||||
errorTest(
|
||||
"distinct window function",
|
||||
CatalystSqlParser.parsePlan("SELECT percent_rank(DISTINCT a) OVER () FROM TaBlE"),
|
||||
"DISTINCT or FILTER specified, but percent_rank is not an aggregate function" :: Nil)
|
||||
"Function percent_rank does not support DISTINCT" :: Nil)
|
||||
|
||||
errorTest(
|
||||
"window function with filter predicate",
|
||||
CatalystSqlParser.parsePlan("SELECT percent_rank(a) FILTER (WHERE c > 1) OVER () FROM TaBlE2"),
|
||||
"DISTINCT or FILTER specified, but percent_rank is not an aggregate function" :: Nil)
|
||||
"Function percent_rank does not support FILTER clause" :: Nil)
|
||||
|
||||
errorTest(
|
||||
"higher order function with filter predicate",
|
||||
|
|
|
|||
Loading…
Reference in a new issue