From f0090463a8601d9137b668bc139b293c2b15fdc1 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 26 Apr 2021 16:42:07 +0000 Subject: [PATCH] [SPARK-33985][SQL][TESTS] Add query test of combine usage of TRANSFORM and CLUSTER BY/ORDER BY ### What changes were proposed in this pull request? Under hive's document https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Transform there are many usage about TRANSFORM and CLUSTER BY/ORDER BY, in this pr add some test about this cases. ### Why are the changes needed? Add UT ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added UT Closes #32333 from AngersZhuuuu/SPARK-33985. Authored-by: Angerszhuuuu Signed-off-by: Wenchen Fan --- .../resources/sql-tests/inputs/transform.sql | 31 +++++++++ .../sql-tests/results/transform.sql.out | 66 ++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/transform.sql b/sql/core/src/test/resources/sql-tests/inputs/transform.sql index 7419ca1bd0..d84659c4cc 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/transform.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/transform.sql @@ -11,6 +11,18 @@ CREATE OR REPLACE TEMPORARY VIEW script_trans AS SELECT * FROM VALUES (7, 8, 9) AS script_trans(a, b, c); +CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES +(1, 1), +(1, 1), +(2, 2), +(2, 2), +(3, 3), +(2, 2), +(3, 3), +(1, 1), +(3, 3) +as complex_trans(a, b); + SELECT TRANSFORM(a) USING 'cat' AS (a) FROM t; @@ -342,3 +354,22 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING)) FROM script_trans WHERE a <= 2 GROUP BY b; + +-- SPARK-33985: TRANSFORM with CLUSTER BY/ORDER BY/SORT BY +FROM ( + SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) + FROM complex_trans + CLUSTER BY a +) map_output +SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b); + +FROM ( + SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) + FROM complex_trans + ORDER BY a +) map_output +SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b); \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index 1d7e9cdb43..6f94e742b8 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 44 +-- Number of queries: 47 -- !query @@ -26,6 +26,24 @@ struct<> +-- !query +CREATE OR REPLACE TEMPORARY VIEW complex_trans AS SELECT * FROM VALUES +(1, 1), +(1, 1), +(2, 2), +(2, 2), +(3, 3), +(2, 2), +(3, 3), +(1, 1), +(3, 3) +as complex_trans(a, b) +-- !query schema +struct<> +-- !query output + + + -- !query SELECT TRANSFORM(a) USING 'cat' AS (a) @@ -717,3 +735,49 @@ SELECT TRANSFORM(b, MAX(a) AS max_a, CAST(sum(c) AS STRING)) FROM script_trans WHERE a <= 2 GROUP BY b + + +-- !query +FROM ( + SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) + FROM complex_trans + CLUSTER BY a +) map_output +SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) +-- !query schema +struct +-- !query output +1 1 +1 1 +1 1 +2 2 +2 2 +2 2 +3 3 +3 3 +3 3 + + +-- !query +FROM ( + SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) + FROM complex_trans + ORDER BY a +) map_output +SELECT TRANSFORM(a, b) + USING 'cat' AS (a, b) +-- !query schema +struct +-- !query output +1 1 +1 1 +1 1 +2 2 +2 2 +2 2 +3 3 +3 3 +3 3