ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-36863][BUILD] Update dependency manifest files for all released Spark artifacts

Browse source 
### What changes were proposed in this pull request?

Update `dev/test-dependencies.sh` so that it now covers all released Spark artifacts such as `hadoop-cloud`.

### Why are the changes needed?

Currently `dev/test-dependencies.sh` doesn't cover all released Spark modules. Therefore, if there is any dependency change in those modules (e.g., `hadoop-cloud`), we won't be able to detect it early.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

N/A

Closes #34119 from sunchao/SPARK-36863-dependency.

Authored-by: Chao Sun <sunchao@apple.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
This commit is contained in:
Chao Sun 2021-09-28 19:22:48 +09:00 committed by Hyukjin Kwon
parent 4caf71f0d5
commit 8d8b71058c
3 changed files with 38 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
dev/deps/spark-deps-hadoop-2.7-hive-2.3
View file

@ -26,6 +26,7 @@ automaton/1.11-8//automaton-1.11-8.jar
avro-ipc/1.10.2//avro-ipc-1.10.2.jar avro-ipc/1.10.2//avro-ipc-1.10.2.jar
avro-mapred/1.10.2//avro-mapred-1.10.2.jar avro-mapred/1.10.2//avro-mapred-1.10.2.jar
avro/1.10.2//avro-1.10.2.jar avro/1.10.2//avro-1.10.2.jar
azure-storage/2.0.0//azure-storage-2.0.0.jar
blas/2.2.0//blas-2.2.0.jar blas/2.2.0//blas-2.2.0.jar
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar
@ -64,12 +65,15 @@ derby/10.14.2.0//derby-10.14.2.0.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
generex/1.0.2//generex-1.0.2.jar generex/1.0.2//generex-1.0.2.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
gson/2.2.4//gson-2.2.4.jar gson/2.2.4//gson-2.2.4.jar
guava/14.0.1//guava-14.0.1.jar guava/14.0.1//guava-14.0.1.jar
guice-servlet/3.0//guice-servlet-3.0.jar guice-servlet/3.0//guice-servlet-3.0.jar
guice/3.0//guice-3.0.jar guice/3.0//guice-3.0.jar
hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar
hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar
hadoop-aws/2.7.4//hadoop-aws-2.7.4.jar
hadoop-azure/2.7.4//hadoop-azure-2.7.4.jar
hadoop-client/2.7.4//hadoop-client-2.7.4.jar hadoop-client/2.7.4//hadoop-client-2.7.4.jar
hadoop-common/2.7.4//hadoop-common-2.7.4.jar hadoop-common/2.7.4//hadoop-common-2.7.4.jar
hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar
@ -78,6 +82,7 @@ hadoop-mapreduce-client-common/2.7.4//hadoop-mapreduce-client-common-2.7.4.jar
hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar
hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar
hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar
hadoop-openstack/2.7.4//hadoop-openstack-2.7.4.jar
hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar
hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
@ -110,6 +115,7 @@ jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.12.5//jackson-core-2.12.5.jar jackson-core/2.12.5//jackson-core-2.12.5.jar
jackson-databind/2.12.5//jackson-databind-2.12.5.jar jackson-databind/2.12.5//jackson-databind-2.12.5.jar
jackson-dataformat-cbor/2.12.5//jackson-dataformat-cbor-2.12.5.jar
jackson-dataformat-yaml/2.12.5//jackson-dataformat-yaml-2.12.5.jar jackson-dataformat-yaml/2.12.5//jackson-dataformat-yaml-2.12.5.jar
jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
@ -139,6 +145,7 @@ jersey-hk2/2.34//jersey-hk2-2.34.jar
jersey-server/2.34//jersey-server-2.34.jar jersey-server/2.34//jersey-server-2.34.jar
jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar
jetty-util/6.1.26//jetty-util-6.1.26.jar jetty-util/6.1.26//jetty-util-6.1.26.jar
jetty-util/9.4.43.v20210629//jetty-util-9.4.43.v20210629.jar
jetty/6.1.26//jetty-6.1.26.jar jetty/6.1.26//jetty-6.1.26.jar
jline/2.14.6//jline-2.14.6.jar jline/2.14.6//jline-2.14.6.jar
joda-time/2.10.10//joda-time-2.10.10.jar joda-time/2.10.10//joda-time-2.10.10.jar
@ -210,6 +217,7 @@ parquet-jackson/1.12.1//parquet-jackson-1.12.1.jar
protobuf-java/2.5.0//protobuf-java-2.5.0.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar
py4j/0.10.9.2//py4j-0.10.9.2.jar py4j/0.10.9.2//py4j-0.10.9.2.jar
pyrolite/4.30//pyrolite-4.30.jar pyrolite/4.30//pyrolite-4.30.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar
scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
scala-compiler/2.12.15//scala-compiler-2.12.15.jar scala-compiler/2.12.15//scala-compiler-2.12.15.jar

26
dev/deps/spark-deps-hadoop-3.2-hive-2.3
View file

@ -6,6 +6,11 @@ ST4/4.0.4//ST4-4.0.4.jar
activation/1.1.1//activation-1.1.1.jar activation/1.1.1//activation-1.1.1.jar
aircompressor/0.21//aircompressor-0.21.jar aircompressor/0.21//aircompressor-0.21.jar
algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar
aliyun-java-sdk-core/3.4.0//aliyun-java-sdk-core-3.4.0.jar
aliyun-java-sdk-ecs/4.2.0//aliyun-java-sdk-ecs-4.2.0.jar
aliyun-java-sdk-ram/3.0.0//aliyun-java-sdk-ram-3.0.0.jar
aliyun-java-sdk-sts/3.0.0//aliyun-java-sdk-sts-3.0.0.jar
aliyun-sdk-oss/3.4.1//aliyun-sdk-oss-3.4.1.jar
annotations/17.0.0//annotations-17.0.0.jar annotations/17.0.0//annotations-17.0.0.jar
antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.8//antlr4-runtime-4.8.jar antlr4-runtime/4.8//antlr4-runtime-4.8.jar
@ -21,6 +26,10 @@ automaton/1.11-8//automaton-1.11-8.jar
avro-ipc/1.10.2//avro-ipc-1.10.2.jar avro-ipc/1.10.2//avro-ipc-1.10.2.jar
avro-mapred/1.10.2//avro-mapred-1.10.2.jar avro-mapred/1.10.2//avro-mapred-1.10.2.jar
avro/1.10.2//avro-1.10.2.jar avro/1.10.2//avro-1.10.2.jar
aws-java-sdk-bundle/1.11.901//aws-java-sdk-bundle-1.11.901.jar
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
azure-storage/7.0.1//azure-storage-7.0.1.jar
blas/2.2.0//blas-2.2.0.jar blas/2.2.0//blas-2.2.0.jar
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar breeze-macros_2.12/1.2//breeze-macros_2.12-1.2.jar
@ -45,6 +54,7 @@ commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-text/1.6//commons-text-1.6.jar commons-text/1.6//commons-text-1.6.jar
compress-lzf/1.0.3//compress-lzf-1.0.3.jar compress-lzf/1.0.3//compress-lzf-1.0.3.jar
core/1.1.2//core-1.1.2.jar core/1.1.2//core-1.1.2.jar
cos_api-bundle/5.6.19//cos_api-bundle-5.6.19.jar
curator-client/2.13.0//curator-client-2.13.0.jar curator-client/2.13.0//curator-client-2.13.0.jar
curator-framework/2.13.0//curator-framework-2.13.0.jar curator-framework/2.13.0//curator-framework-2.13.0.jar
curator-recipes/2.13.0//curator-recipes-2.13.0.jar curator-recipes/2.13.0//curator-recipes-2.13.0.jar
@ -55,10 +65,19 @@ derby/10.14.2.0//derby-10.14.2.0.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar
generex/1.0.2//generex-1.0.2.jar generex/1.0.2//generex-1.0.2.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
gson/2.2.4//gson-2.2.4.jar gson/2.2.4//gson-2.2.4.jar
guava/14.0.1//guava-14.0.1.jar guava/14.0.1//guava-14.0.1.jar
hadoop-aliyun/3.3.1//hadoop-aliyun-3.3.1.jar
hadoop-annotations/3.3.1//hadoop-annotations-3.3.1.jar
hadoop-aws/3.3.1//hadoop-aws-3.3.1.jar
hadoop-azure-datalake/3.3.1//hadoop-azure-datalake-3.3.1.jar
hadoop-azure/3.3.1//hadoop-azure-3.3.1.jar
hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar hadoop-client-api/3.3.1//hadoop-client-api-3.3.1.jar
hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar hadoop-client-runtime/3.3.1//hadoop-client-runtime-3.3.1.jar
hadoop-cloud-storage/3.3.1//hadoop-cloud-storage-3.3.1.jar
hadoop-cos/3.3.1//hadoop-cos-3.3.1.jar
hadoop-openstack/3.3.1//hadoop-openstack-3.3.1.jar
hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar hadoop-yarn-server-web-proxy/3.3.1//hadoop-yarn-server-web-proxy-3.3.1.jar
hive-beeline/2.3.9//hive-beeline-2.3.9.jar hive-beeline/2.3.9//hive-beeline-2.3.9.jar
@ -88,6 +107,7 @@ jackson-annotations/2.12.5//jackson-annotations-2.12.5.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.12.5//jackson-core-2.12.5.jar jackson-core/2.12.5//jackson-core-2.12.5.jar
jackson-databind/2.12.5//jackson-databind-2.12.5.jar jackson-databind/2.12.5//jackson-databind-2.12.5.jar
jackson-dataformat-cbor/2.12.5//jackson-dataformat-cbor-2.12.5.jar
jackson-dataformat-yaml/2.12.5//jackson-dataformat-yaml-2.12.5.jar jackson-dataformat-yaml/2.12.5//jackson-dataformat-yaml-2.12.5.jar
jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar jackson-datatype-jsr310/2.11.2//jackson-datatype-jsr310-2.11.2.jar
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
@ -105,12 +125,16 @@ javolution/5.5.1//javolution-5.5.1.jar
jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
jdo-api/3.0.1//jdo-api-3.0.1.jar jdo-api/3.0.1//jdo-api-3.0.1.jar
jdom/1.1//jdom-1.1.jar
jersey-client/2.34//jersey-client-2.34.jar jersey-client/2.34//jersey-client-2.34.jar
jersey-common/2.34//jersey-common-2.34.jar jersey-common/2.34//jersey-common-2.34.jar
jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar
jersey-hk2/2.34//jersey-hk2-2.34.jar jersey-hk2/2.34//jersey-hk2-2.34.jar
jersey-server/2.34//jersey-server-2.34.jar jersey-server/2.34//jersey-server-2.34.jar
jettison/1.1//jettison-1.1.jar
jetty-util-ajax/9.4.43.v20210629//jetty-util-ajax-9.4.43.v20210629.jar
jetty-util/9.4.43.v20210629//jetty-util-9.4.43.v20210629.jar
jline/2.14.6//jline-2.14.6.jar jline/2.14.6//jline-2.14.6.jar
joda-time/2.10.10//joda-time-2.10.10.jar joda-time/2.10.10//joda-time-2.10.10.jar
jodd-core/3.5.2//jodd-core-3.5.2.jar jodd-core/3.5.2//jodd-core-3.5.2.jar
@ -180,6 +204,7 @@ parquet-jackson/1.12.1//parquet-jackson-1.12.1.jar
protobuf-java/2.5.0//protobuf-java-2.5.0.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar
py4j/0.10.9.2//py4j-0.10.9.2.jar py4j/0.10.9.2//py4j-0.10.9.2.jar
pyrolite/4.30//pyrolite-4.30.jar pyrolite/4.30//pyrolite-4.30.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar
scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
scala-compiler/2.12.15//scala-compiler-2.12.15.jar scala-compiler/2.12.15//scala-compiler-2.12.15.jar
@ -205,6 +230,7 @@ tink/1.6.0//tink-1.6.0.jar
transaction-api/1.1//transaction-api-1.1.jar transaction-api/1.1//transaction-api-1.1.jar
univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
velocity/1.5//velocity-1.5.jar velocity/1.5//velocity-1.5.jar
wildfly-openssl/1.0.7.Final//wildfly-openssl-1.0.7.Final.jar
xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar xbean-asm9-shaded/4.20//xbean-asm9-shaded-4.20.jar
xz/1.8//xz-1.8.jar xz/1.8//xz-1.8.jar
zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar

6
dev/test-dependencies.sh
View file

@ -28,8 +28,10 @@ export LC_ALL=C
# TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution. # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
# NOTE: These should match those in the release publishing script # NOTE: These should match those in the release publishing script, and be kept in sync with
HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive" # dev/create-release/release-build.sh
HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive \
-Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud"
MVN="build/mvn" MVN="build/mvn"
HADOOP_HIVE_PROFILES=( HADOOP_HIVE_PROFILES=(
hadoop-2.7-hive-2.3 hadoop-2.7-hive-2.3