2014-07-28 15:07:30 -04:00
|
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
|
|
<!--
|
|
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
|
|
~ this work for additional information regarding copyright ownership.
|
|
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
|
|
~ the License. You may obtain a copy of the License at
|
|
|
|
~
|
|
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
~
|
|
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
~ See the License for the specific language governing permissions and
|
|
|
|
~ limitations under the License.
|
|
|
|
-->
|
|
|
|
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
2018-11-14 19:22:23 -05:00
|
|
|
<artifactId>spark-parent_2.12</artifactId>
|
2020-12-04 17:10:42 -05:00
|
|
|
<version>3.2.0-SNAPSHOT</version>
|
2014-07-28 15:07:30 -04:00
|
|
|
<relativePath>../../pom.xml</relativePath>
|
|
|
|
</parent>
|
|
|
|
|
2018-11-14 19:22:23 -05:00
|
|
|
<artifactId>spark-hive-thriftserver_2.12</artifactId>
|
2014-07-28 15:07:30 -04:00
|
|
|
<packaging>jar</packaging>
|
2014-08-05 21:30:02 -04:00
|
|
|
<name>Spark Project Hive Thrift Server</name>
|
2014-07-28 15:07:30 -04:00
|
|
|
<url>http://spark.apache.org/</url>
|
|
|
|
<properties>
|
|
|
|
<sbt.project.name>hive-thriftserver</sbt.project.name>
|
|
|
|
</properties>
|
|
|
|
|
|
|
|
<dependencies>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
</dependency>
|
2015-05-29 17:03:12 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2019-08-26 09:39:57 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2019-06-02 23:23:08 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2015-03-05 03:35:17 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.google.guava</groupId>
|
|
|
|
<artifactId>guava</artifactId>
|
|
|
|
</dependency>
|
2014-07-28 15:07:30 -04:00
|
|
|
<dependency>
|
2015-01-08 20:15:13 -05:00
|
|
|
<groupId>${hive.group}</groupId>
|
2014-07-28 15:07:30 -04:00
|
|
|
<artifactId>hive-cli</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
2015-01-08 20:15:13 -05:00
|
|
|
<groupId>${hive.group}</groupId>
|
2014-07-28 15:07:30 -04:00
|
|
|
<artifactId>hive-jdbc</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
2015-01-08 20:15:13 -05:00
|
|
|
<groupId>${hive.group}</groupId>
|
2014-07-28 15:07:30 -04:00
|
|
|
<artifactId>hive-beeline</artifactId>
|
|
|
|
</dependency>
|
2020-10-16 12:37:54 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>${hive.group}</groupId>
|
|
|
|
<artifactId>hive-service-rpc</artifactId>
|
|
|
|
</dependency>
|
2017-09-24 04:40:13 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-server</artifactId>
|
|
|
|
<scope>provided</scope>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-servlet</artifactId>
|
|
|
|
<scope>provided</scope>
|
|
|
|
</dependency>
|
2015-05-04 04:59:34 -04:00
|
|
|
<!-- Added for selenium: -->
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.seleniumhq.selenium</groupId>
|
|
|
|
<artifactId>selenium-java</artifactId>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2016-05-15 10:56:46 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.seleniumhq.selenium</groupId>
|
2020-06-11 19:27:53 -04:00
|
|
|
<artifactId>htmlunit-driver</artifactId>
|
2016-05-15 10:56:46 -04:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2015-08-18 23:21:46 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2015-10-07 17:11:21 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
2016-05-17 04:55:53 -04:00
|
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
2015-10-07 17:11:21 -04:00
|
|
|
</dependency>
|
2016-12-21 19:37:20 -05:00
|
|
|
|
|
|
|
<!--
|
|
|
|
This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
|
|
|
|
them will yield errors.
|
|
|
|
-->
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2019-11-06 06:59:45 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.mockito</groupId>
|
|
|
|
<artifactId>mockito-core</artifactId>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2016-04-29 12:32:42 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>net.sf.jpam</groupId>
|
|
|
|
<artifactId>jpam</artifactId>
|
|
|
|
</dependency>
|
2021-03-17 23:31:50 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>commons-cli</groupId>
|
|
|
|
<artifactId>commons-cli</artifactId>
|
|
|
|
</dependency>
|
2014-07-28 15:07:30 -04:00
|
|
|
</dependencies>
|
[SPARK-35838][BUILD][TESTS] Ensure all modules can be maven test independently in Scala 2.13
### What changes were proposed in this pull request?
Similar to SPARK-35532, the main change of this pr is add `scala-2.13` profile to external/kafka-0-10-sql/pom.xml, external/avro/pom.xml and sql/hive-thriftserver/pom.xml, the `scala-2.13` profile include dependency on `scala-parallel-collections_2.13`, then all(34) spark modules can maven test independently.
### Why are the changes needed?
Ensure alll(34) spark modules can be maven test independently in Scala 2.13
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass the GitHub Action Scala 2.13 job
- Manual test:
1. Execute
```
dev/change-scala-version.sh 2.13
mvn clean install -DskipTests -Phadoop-3.2 -Phive-2.3 -Phadoop-cloud -Pmesos -Pyarn -Pkinesis-asl -Phive-thriftserver -Pspark-ganglia-lgpl -Pkubernetes -Phive -Pscala-2.13
```
2. maven test `external/kafka-0-10-sql` module
```
mvn test -Phadoop-3.2 -Phive-2.3 -Phadoop-cloud -Pmesos -Pyarn -Pkinesis-asl -Phive-thriftserver -Pspark-ganglia-lgpl -Pkubernetes -Phive -Pscala-2.13 -pl external/kafka-0-10-sql
```
**before**
```
Discovery starting.
Discovery completed in 857 milliseconds.
Run starting. Expected test count is: 464
...
KafkaRelationSuiteV2:
- explicit earliest to latest offsets
- default starting and ending offsets
- explicit offsets
- default starting and ending offsets with headers
- timestamp provided for starting and ending
- timestamp provided for starting, offset provided for ending
- timestamp provided for ending, offset provided for starting
- timestamp provided for starting, ending not provided
- timestamp provided for ending, starting not provided
- global timestamp provided for starting and ending
- no matched offset for timestamp - startingOffsets
- preferences on offset related options
- no matched offset for timestamp - endingOffsets
*** RUN ABORTED ***
java.lang.NoClassDefFoundError: scala/collection/parallel/TaskSupport
at org.apache.spark.SparkContext.$anonfun$union$1(SparkContext.scala:1411)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.SparkContext.withScope(SparkContext.scala:788)
at org.apache.spark.SparkContext.union(SparkContext.scala:1405)
at org.apache.spark.sql.execution.UnionExec.doExecute(basicPhysicalOperators.scala:697)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:182)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:220)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:217)
...
Cause: java.lang.ClassNotFoundException: scala.collection.parallel.TaskSupport
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at org.apache.spark.SparkContext.$anonfun$union$1(SparkContext.scala:1411)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.SparkContext.withScope(SparkContext.scala:788)
at org.apache.spark.SparkContext.union(SparkContext.scala:1405)
at org.apache.spark.sql.execution.UnionExec.doExecute(basicPhysicalOperators.scala:697)
...
```
**After**
```
Run completed in 33 minutes, 51 seconds.
Total number of tests run: 464
Suites: completed 31, aborted 0
Tests: succeeded 464, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```
3. maven test `external/avro` module
```
mvn test -Phadoop-3.2 -Phive-2.3 -Phadoop-cloud -Pmesos -Pyarn -Pkinesis-asl -Phive-thriftserver -Pspark-ganglia-lgpl -Pkubernetes -Phive -Pscala-2.13 -pl external/avro
```
**before**
```
Discovery starting.
Discovery completed in 2 seconds, 765 milliseconds.
Run starting. Expected test count is: 255
AvroReadSchemaSuite:
- append column at the end
- hide column at the end
- append column into middle
- hide column in the middle
- add a nested column at the end of the leaf struct column
- add a nested column in the middle of the leaf struct column
- add a nested column at the end of the middle struct column
- add a nested column in the middle of the middle struct column
- hide a nested column at the end of the leaf struct column
- hide a nested column in the middle of the leaf struct column
- hide a nested column at the end of the middle struct column
- hide a nested column in the middle of the middle struct column
*** RUN ABORTED ***
java.lang.NoClassDefFoundError: scala/collection/parallel/TaskSupport
at org.apache.spark.SparkContext.$anonfun$union$1(SparkContext.scala:1411)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.SparkContext.withScope(SparkContext.scala:788)
at org.apache.spark.SparkContext.union(SparkContext.scala:1405)
at org.apache.spark.sql.execution.UnionExec.doExecute(basicPhysicalOperators.scala:697)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:182)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:220)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:217)
...
Cause: java.lang.ClassNotFoundException: scala.collection.parallel.TaskSupport
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at org.apache.spark.SparkContext.$anonfun$union$1(SparkContext.scala:1411)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.SparkContext.withScope(SparkContext.scala:788)
at org.apache.spark.SparkContext.union(SparkContext.scala:1405)
at org.apache.spark.sql.execution.UnionExec.doExecute(basicPhysicalOperators.scala:697)
...
```
**After**
```
Run completed in 1 minute, 42 seconds.
Total number of tests run: 255
Suites: completed 12, aborted 0
Tests: succeeded 255, failed 0, canceled 0, ignored 2, pending 0
All tests passed.
```
4. maven test `sql/hive-thriftserver` module
```
mvn test -Phadoop-3.2 -Phive-2.3 -Phadoop-cloud -Pmesos -Pyarn -Pkinesis-asl -Phive-thriftserver -Pspark-ganglia-lgpl -Pkubernetes -Phive -Pscala-2.13 -pl sql/hive-thriftserver
```
**before**
```
- union.sql *** FAILED ***
"1 a
1 a
2 b
2 b" did not contain "Exception" Exception did not match for query #2
SELECT *
FROM (SELECT * FROM t1
UNION ALL
SELECT * FROM t1), expected: 1 a
1 a
2 b
2 b, but got: java.sql.SQLException
org.apache.hive.service.cli.HiveSQLException: Error running query: java.lang.NoClassDefFoundError: scala/collection/parallel/TaskSupport
at org.apache.spark.sql.hive.thriftserver.HiveThriftServerErrors$.runningQueryError(HiveThriftServerErrors.scala:38)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:324)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:229)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:79)
at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:63)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:43)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:229)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:224)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:238)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NoClassDefFoundError: scala/collection/parallel/TaskSupport
at org.apache.spark.SparkContext.$anonfun$union$1(SparkContext.scala:1411)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.SparkContext.withScope(SparkContext.scala:788)
at org.apache.spark.SparkContext.union(SparkContext.scala:1405)
at org.apache.spark.sql.execution.UnionExec.doExecute(basicPhysicalOperators.scala:697)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:182)
at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:220)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:217)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:178)
at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:323)
at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:389)
at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3719)
at org.apache.spark.sql.Dataset.$anonfun$collect$1(Dataset.scala:2987)
at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3710)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:774)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3708)
at org.apache.spark.sql.Dataset.collect(Dataset.scala:2987)
at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:299)
... 16 more
Caused by: java.lang.ClassNotFoundException: scala.collection.parallel.TaskSupport
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 40 more (ThriftServerQueryTestSuite.scala:209)
```
**After**
```
Run completed in 29 minutes, 17 seconds.
Total number of tests run: 535
Suites: completed 20, aborted 0
Tests: succeeded 535, failed 0, canceled 0, ignored 17, pending 0
All tests passed.
```
Closes #32994 from LuciferYang/SPARK-35838.
Authored-by: YangJie <yangjie01@baidu.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
2021-06-22 09:31:24 -04:00
|
|
|
<profiles>
|
|
|
|
<profile>
|
|
|
|
<id>scala-2.13</id>
|
|
|
|
<dependencies>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.scala-lang.modules</groupId>
|
|
|
|
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
|
|
|
</dependency>
|
|
|
|
</dependencies>
|
|
|
|
</profile>
|
|
|
|
</profiles>
|
2014-07-28 15:07:30 -04:00
|
|
|
<build>
|
|
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
|
|
</build>
|
|
|
|
</project>
|