[SPARK-30783] Exclude hive-service-rpc
### What changes were proposed in this pull request? Exclude hive-service-rpc from build. ### Why are the changes needed? hive-service-rpc 2.3.6 and spark sql's thrift server module have duplicate classes. Leaving hive-service-rpc 2.3.6 in the class path means that spark can pick up classes defined in hive instead of its thrift server module, which can cause hard to debug runtime errors due to class loading order and compilation errors for applications depend on spark. If you compare hive-service-rpc 2.3.6's jar (https://search.maven.org/remotecontent?filepath=org/apache/hive/hive-service-rpc/2.3.6/hive-service-rpc-2.3.6.jar) and spark thrift server's jar (e.g. https://repository.apache.org/content/groups/snapshots/org/apache/spark/spark-hive-thriftserver_2.12/3.0.0-SNAPSHOT/spark-hive-thriftserver_2.12-3.0.0-20200207.021914-364.jar), you will see that all of classes provided by hive-service-rpc-2.3.6.jar are covered by spark thrift server's jar. https://issues.apache.org/jira/browse/SPARK-30783 has output of jar tf for both jars. ### Does this PR introduce any user-facing change? No ### How was this patch tested? Existing tests. Closes #27533 from yhuai/SPARK-30783. Authored-by: Yin Huai <yhuai@databricks.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
dc66d57e98
commit
ea626b6acf
|
@ -87,7 +87,6 @@ hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
|
||||||
hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
|
hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
|
||||||
hive-metastore/2.3.6//hive-metastore-2.3.6.jar
|
hive-metastore/2.3.6//hive-metastore-2.3.6.jar
|
||||||
hive-serde/2.3.6//hive-serde-2.3.6.jar
|
hive-serde/2.3.6//hive-serde-2.3.6.jar
|
||||||
hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
|
|
||||||
hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
|
hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
|
||||||
hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
|
hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
|
||||||
hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
|
hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
|
||||||
|
|
|
@ -86,7 +86,6 @@ hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
|
||||||
hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
|
hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
|
||||||
hive-metastore/2.3.6//hive-metastore-2.3.6.jar
|
hive-metastore/2.3.6//hive-metastore-2.3.6.jar
|
||||||
hive-serde/2.3.6//hive-serde-2.3.6.jar
|
hive-serde/2.3.6//hive-serde-2.3.6.jar
|
||||||
hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
|
|
||||||
hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
|
hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
|
||||||
hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
|
hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
|
||||||
hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
|
hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
|
||||||
|
|
20
pom.xml
20
pom.xml
|
@ -1452,6 +1452,11 @@
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-service</artifactId>
|
<artifactId>hive-service</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<!-- All classes are covered by spark's hive-thriftserver module -->
|
||||||
|
<groupId>${hive.group}</groupId>
|
||||||
|
<artifactId>hive-service-rpc</artifactId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-shims</artifactId>
|
<artifactId>hive-shims</artifactId>
|
||||||
|
@ -1508,6 +1513,11 @@
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-service</artifactId>
|
<artifactId>hive-service</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<!-- All classes are covered by spark's hive-thriftserver module -->
|
||||||
|
<groupId>${hive.group}</groupId>
|
||||||
|
<artifactId>hive-service-rpc</artifactId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-shims</artifactId>
|
<artifactId>hive-shims</artifactId>
|
||||||
|
@ -1761,6 +1771,11 @@
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-service</artifactId>
|
<artifactId>hive-service</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<!-- All classes are covered by spark's hive-thriftserver module -->
|
||||||
|
<groupId>${hive.group}</groupId>
|
||||||
|
<artifactId>hive-service-rpc</artifactId>
|
||||||
|
</exclusion>
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>${hive.group}</groupId>
|
<groupId>${hive.group}</groupId>
|
||||||
<artifactId>hive-shims</artifactId>
|
<artifactId>hive-shims</artifactId>
|
||||||
|
@ -1911,6 +1926,11 @@
|
||||||
<artifactId>groovy-all</artifactId>
|
<artifactId>groovy-all</artifactId>
|
||||||
</exclusion>
|
</exclusion>
|
||||||
<!-- Begin of Hive 2.3 exclusion -->
|
<!-- Begin of Hive 2.3 exclusion -->
|
||||||
|
<exclusion>
|
||||||
|
<!-- All classes are covered by spark's hive-thriftserver module -->
|
||||||
|
<groupId>${hive.group}</groupId>
|
||||||
|
<artifactId>hive-service-rpc</artifactId>
|
||||||
|
</exclusion>
|
||||||
<!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
|
<!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
|
||||||
<exclusion>
|
<exclusion>
|
||||||
<groupId>org.apache.parquet</groupId>
|
<groupId>org.apache.parquet</groupId>
|
||||||
|
|
Loading…
Reference in a new issue