ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-10437] [SQL] Support aggregation expressions in Order By

Browse source 
JIRA: https://issues.apache.org/jira/browse/SPARK-10437

If an expression in `SortOrder` is a resolved one, such as `count(1)`, the corresponding rule in `Analyzer` to make it work in order by will not be applied.

Author: Liang-Chi Hsieh <viirya@appier.com>

Closes #8599 from viirya/orderby-agg.
This commit is contained in:
Liang-Chi Hsieh 2015-09-15 13:33:32 -07:00 committed by Michael Armbrust
parent b42059d2ef
commit 841972e22c
2 changed files with 30 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
View file

@ -561,7 +561,7 @@ class Analyzer(
}
case sort @ Sort(sortOrder, global, aggregate: Aggregate)
if aggregate.resolved && !sort.resolved =>
if aggregate.resolved =>
// Try resolving the ordering as though it is in the aggregate clause.
try {
@ -598,9 +598,15 @@ class Analyzer(
}
}
Project(aggregate.output,
Sort(evaluatedOrderings, global,
aggregate.copy(aggregateExpressions = originalAggExprs ++ needsPushDown)))
// Since we don't rely on sort.resolved as the stop condition for this rule,
// we need to check this and prevent applying this rule multiple times
if (sortOrder == evaluatedOrderings) {
sort
} else {
Project(aggregate.output,
Sort(evaluatedOrderings, global,
aggregate.copy(aggregateExpressions = originalAggExprs ++ needsPushDown)))
}
} catch {
// Attempting to resolve in the aggregate can result in ambiguity. When this happens,
// just return the original plan.

20
sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
View file

@ -1562,6 +1562,26 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
|ORDER BY sum(b) + 1
""".stripMargin),
Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
checkAnswer(
sql(
"""
|SELECT count(*)
|FROM orderByData
|GROUP BY a
|ORDER BY count(*)
""".stripMargin),
Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
checkAnswer(
sql(
"""
|SELECT a
|FROM orderByData
|GROUP BY a
|ORDER BY a, count(*), sum(b)
""".stripMargin),
Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
}
test("SPARK-7952: fix the equality check between boolean and numeric types") {