ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-33985][SQL][TESTS] Add query test of combine usage of TRANSFORM and CLUSTER BY/ORDER BY

Browse source 
### What changes were proposed in this pull request?
Under hive's document  https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Transform there are many usage about  TRANSFORM and CLUSTER BY/ORDER BY, in this pr add some test about this cases.

### Why are the changes needed?
Add UT

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT

Closes #32333 from AngersZhuuuu/SPARK-33985.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
Angerszhuuuu 2021-04-26 16:42:07 +00:00 committed by Wenchen Fan
parent c59988aa79
commit f0090463a8
2 changed files with 96 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
sql/core/src/test/resources/sql-tests/inputs/transform.sql
View file

@ -11,6 +11,18 @@ CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES
(7, 8, 9)
AS script_trans(a, b, c);
CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
(1, 1),
(1, 1),
(2, 2),
(2, 2),
(3, 3),
(2, 2),
(3, 3),
(1, 1),
(3, 3)
as complex_trans(a, b);
SELECT TRANSFORM(a)
USING 'cat' AS (a)
FROM t;
@ -342,3 +354,22 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
FROM script_trans
WHERE a <= 2
GROUP BY b;
-- SPARK-33985: TRANSFORM with CLUSTER BY/ORDER BY/SORT BY
FROM (
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
FROM complex_trans
CLUSTER BY a
) map_output
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b);
FROM (
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
FROM complex_trans
ORDER BY a
) map_output
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b);

66
sql/core/src/test/resources/sql-tests/results/transform.sql.out
View file

@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 44
-- Number of queries: 47
-- !query
@ -26,6 +26,24 @@ struct<>
-- !query
CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES
(1, 1),
(1, 1),
(2, 2),
(2, 2),
(3, 3),
(2, 2),
(3, 3),
(1, 1),
(3, 3)
as complex_trans(a, b)
-- !query schema
struct<>
-- !query output
-- !query
SELECT TRANSFORM(a)
USING 'cat' AS (a)
@ -717,3 +735,49 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING))
FROM script_trans
WHERE a <= 2
GROUP BY b
-- !query
FROM (
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
FROM complex_trans
CLUSTER BY a
) map_output
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
-- !query schema
struct<a:string,b:string>
-- !query output
1 1
1 1
1 1
2 2
2 2
2 2
3 3
3 3
3 3
-- !query
FROM (
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
FROM complex_trans
ORDER BY a
) map_output
SELECT TRANSFORM(a, b)
USING 'cat' AS (a, b)
-- !query schema
struct<a:string,b:string>
-- !query output
1 1
1 1
1 1
2 2
2 2
2 2
3 3
3 3
3 3