[SPARK-30412][SQL][TESTS] Eliminate warnings in Java tests regarding to deprecated Spark SQL API
### What changes were proposed in this pull request? In the PR, I propose to add the `SuppressWarnings("deprecation")` annotation to Java tests for deprecated Spark SQL APIs. ### Why are the changes needed? This eliminates the following warnings: ``` sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetAggregatorSuite.java Warning:Warning:line (32)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (91)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (100)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (109)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (118)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated sql/core/src/test/java/test/org/apache/spark/sql/Java8DatasetAggregatorSuite.java Warning:Warning:line (28)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (37)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (46)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (55)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated Warning:Warning:line (64)java: org.apache.spark.sql.expressions.javalang.typed in org.apache.spark.sql.expressions.javalang has been deprecated sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java Warning:Warning:line (478)java: json(org.apache.spark.api.java.JavaRDD<java.lang.String>) in org.apache.spark.sql.DataFrameReader has been deprecated ``` and highlights warnings about real problems. ### Does this PR introduce any user-facing change? No ### How was this patch tested? By existing test suites `Java8DatasetAggregatorSuite.java`, `JavaDataFrameSuite.java` and `JavaDatasetAggregatorSuite.java`. Closes #27081 from MaxGekk/eliminate-warnings-part2. Authored-by: Maxim Gekk <max.gekk@gmail.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
parent
a469976e6e
commit
51373467cc
|
@ -25,43 +25,50 @@ import scala.Tuple2;
|
|||
|
||||
import org.apache.spark.sql.Dataset;
|
||||
import org.apache.spark.sql.KeyValueGroupedDataset;
|
||||
import org.apache.spark.sql.expressions.javalang.typed;
|
||||
|
||||
/**
|
||||
* Suite that replicates tests in JavaDatasetAggregatorSuite using lambda syntax.
|
||||
*/
|
||||
public class Java8DatasetAggregatorSuite extends JavaDatasetAggregatorSuiteBase {
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationAverage() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(typed.avg(v -> (double)(v._2() * 2)));
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.avg(v -> (double)(v._2() * 2)));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 6.0)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationCount() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(typed.count(v -> v));
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.count(v -> v));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 2L), new Tuple2<>("b", 1L)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationSumDouble() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(typed.sum(v -> (double)v._2()));
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.sum(v -> (double)v._2()));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 3.0)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationSumLong() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(typed.sumLong(v -> (long)v._2()));
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.sumLong(v -> (long)v._2()));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3L), new Tuple2<>("b", 3L)),
|
||||
agged.collectAsList());
|
||||
|
|
|
@ -471,6 +471,7 @@ public class JavaDataFrameSuite {
|
|||
Assert.assertEquals(1, df.collectAsList().size());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testJsonRDDToDataFrame() {
|
||||
// This is a test for the deprecated API in SPARK-15615.
|
||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.spark.sql.Encoder;
|
|||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.KeyValueGroupedDataset;
|
||||
import org.apache.spark.sql.expressions.Aggregator;
|
||||
import org.apache.spark.sql.expressions.javalang.typed;
|
||||
|
||||
/**
|
||||
* Suite for testing the aggregate functionality of Datasets in Java.
|
||||
|
@ -85,37 +84,45 @@ public class JavaDatasetAggregatorSuite extends JavaDatasetAggregatorSuiteBase {
|
|||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationAverage() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(typed.avg(value -> value._2() * 2.0));
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.avg(value -> value._2() * 2.0));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 6.0)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationCount() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(typed.count(value -> value));
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.count(value -> value));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 2L), new Tuple2<>("b", 1L)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationSumDouble() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(typed.sum(value -> (double) value._2()));
|
||||
Dataset<Tuple2<String, Double>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.sum(value -> (double) value._2()));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3.0), new Tuple2<>("b", 3.0)),
|
||||
agged.collectAsList());
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test
|
||||
public void testTypedAggregationSumLong() {
|
||||
KeyValueGroupedDataset<String, Tuple2<String, Integer>> grouped = generateGroupedDataset();
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(typed.sumLong(value -> (long) value._2()));
|
||||
Dataset<Tuple2<String, Long>> agged = grouped.agg(
|
||||
org.apache.spark.sql.expressions.javalang.typed.sumLong(value -> (long) value._2()));
|
||||
Assert.assertEquals(
|
||||
Arrays.asList(new Tuple2<>("a", 3L), new Tuple2<>("b", 3L)),
|
||||
agged.collectAsList());
|
||||
|
|
Loading…
Reference in a new issue