[SPARK-12612][PROJECT-INFRA] Add missing Hadoop profiles to dev/run-tests-*.py scripts and dev/deps
There are a couple of places in the `dev/run-tests-*.py` scripts which deal with Hadoop profiles, but the set of profiles that they handle does not include all Hadoop profiles defined in our POM. Similarly, the `hadoop-2.2` and `hadoop-2.6` profiles were missing from `dev/deps`. This patch updates these scripts to include all four Hadoop profiles defined in our POM. Author: Josh Rosen <joshrosen@databricks.com> Closes #10565 from JoshRosen/add-missing-hadoop-profiles-in-test-scripts.
This commit is contained in:
parent
84f8492c15
commit
0d165ec205
193
dev/deps/spark-deps-hadoop-2.2
Normal file
193
dev/deps/spark-deps-hadoop-2.2
Normal file
|
@ -0,0 +1,193 @@
|
|||
JavaEWAH-0.3.2.jar
|
||||
RoaringBitmap-0.5.11.jar
|
||||
ST4-4.0.4.jar
|
||||
activation-1.1.jar
|
||||
akka-actor_2.10-2.3.11.jar
|
||||
akka-remote_2.10-2.3.11.jar
|
||||
akka-slf4j_2.10-2.3.11.jar
|
||||
antlr-2.7.7.jar
|
||||
antlr-runtime-3.4.jar
|
||||
aopalliance-1.0.jar
|
||||
apache-log4j-extras-1.2.17.jar
|
||||
arpack_combined_all-0.1.jar
|
||||
asm-3.1.jar
|
||||
asm-commons-3.1.jar
|
||||
asm-tree-3.1.jar
|
||||
avro-1.7.7.jar
|
||||
avro-ipc-1.7.7-tests.jar
|
||||
avro-ipc-1.7.7.jar
|
||||
avro-mapred-1.7.7-hadoop2.jar
|
||||
bonecp-0.8.0.RELEASE.jar
|
||||
breeze-macros_2.10-0.11.2.jar
|
||||
breeze_2.10-0.11.2.jar
|
||||
calcite-avatica-1.2.0-incubating.jar
|
||||
calcite-core-1.2.0-incubating.jar
|
||||
calcite-linq4j-1.2.0-incubating.jar
|
||||
chill-java-0.5.0.jar
|
||||
chill_2.10-0.5.0.jar
|
||||
commons-beanutils-1.7.0.jar
|
||||
commons-beanutils-core-1.8.0.jar
|
||||
commons-cli-1.2.jar
|
||||
commons-codec-1.10.jar
|
||||
commons-collections-3.2.2.jar
|
||||
commons-compiler-2.7.6.jar
|
||||
commons-compress-1.4.1.jar
|
||||
commons-configuration-1.6.jar
|
||||
commons-dbcp-1.4.jar
|
||||
commons-digester-1.8.jar
|
||||
commons-httpclient-3.1.jar
|
||||
commons-io-2.4.jar
|
||||
commons-lang-2.6.jar
|
||||
commons-lang3-3.3.2.jar
|
||||
commons-logging-1.1.3.jar
|
||||
commons-math-2.1.jar
|
||||
commons-math3-3.4.1.jar
|
||||
commons-net-2.2.jar
|
||||
commons-pool-1.5.4.jar
|
||||
compress-lzf-1.0.3.jar
|
||||
config-1.2.1.jar
|
||||
core-1.1.2.jar
|
||||
curator-client-2.4.0.jar
|
||||
curator-framework-2.4.0.jar
|
||||
curator-recipes-2.4.0.jar
|
||||
datanucleus-api-jdo-3.2.6.jar
|
||||
datanucleus-core-3.2.10.jar
|
||||
datanucleus-rdbms-3.2.9.jar
|
||||
derby-10.10.1.1.jar
|
||||
eigenbase-properties-1.1.5.jar
|
||||
geronimo-annotation_1.0_spec-1.1.1.jar
|
||||
geronimo-jaspic_1.0_spec-1.0.jar
|
||||
geronimo-jta_1.1_spec-1.1.1.jar
|
||||
gmbal-api-only-3.0.0-b023.jar
|
||||
grizzly-framework-2.1.2.jar
|
||||
grizzly-http-2.1.2.jar
|
||||
grizzly-http-server-2.1.2.jar
|
||||
grizzly-http-servlet-2.1.2.jar
|
||||
grizzly-rcm-2.1.2.jar
|
||||
groovy-all-2.1.6.jar
|
||||
guice-3.0.jar
|
||||
guice-servlet-3.0.jar
|
||||
hadoop-annotations-2.2.0.jar
|
||||
hadoop-auth-2.2.0.jar
|
||||
hadoop-client-2.2.0.jar
|
||||
hadoop-common-2.2.0.jar
|
||||
hadoop-hdfs-2.2.0.jar
|
||||
hadoop-mapreduce-client-app-2.2.0.jar
|
||||
hadoop-mapreduce-client-common-2.2.0.jar
|
||||
hadoop-mapreduce-client-core-2.2.0.jar
|
||||
hadoop-mapreduce-client-jobclient-2.2.0.jar
|
||||
hadoop-mapreduce-client-shuffle-2.2.0.jar
|
||||
hadoop-yarn-api-2.2.0.jar
|
||||
hadoop-yarn-client-2.2.0.jar
|
||||
hadoop-yarn-common-2.2.0.jar
|
||||
hadoop-yarn-server-common-2.2.0.jar
|
||||
hadoop-yarn-server-web-proxy-2.2.0.jar
|
||||
httpclient-4.3.2.jar
|
||||
httpcore-4.3.2.jar
|
||||
ivy-2.4.0.jar
|
||||
jackson-annotations-2.4.4.jar
|
||||
jackson-core-2.4.4.jar
|
||||
jackson-core-asl-1.9.13.jar
|
||||
jackson-databind-2.4.4.jar
|
||||
jackson-jaxrs-1.9.13.jar
|
||||
jackson-mapper-asl-1.9.13.jar
|
||||
jackson-module-scala_2.10-2.4.4.jar
|
||||
jackson-xc-1.9.13.jar
|
||||
janino-2.7.8.jar
|
||||
jansi-1.4.jar
|
||||
javax.inject-1.jar
|
||||
javax.servlet-3.0.0.v201112011016.jar
|
||||
javax.servlet-3.1.jar
|
||||
javax.servlet-api-3.0.1.jar
|
||||
javolution-5.5.1.jar
|
||||
jaxb-api-2.2.2.jar
|
||||
jaxb-impl-2.2.3-1.jar
|
||||
jcl-over-slf4j-1.7.10.jar
|
||||
jdo-api-3.0.1.jar
|
||||
jersey-client-1.9.jar
|
||||
jersey-core-1.9.jar
|
||||
jersey-grizzly2-1.9.jar
|
||||
jersey-guice-1.9.jar
|
||||
jersey-json-1.9.jar
|
||||
jersey-server-1.9.jar
|
||||
jersey-test-framework-core-1.9.jar
|
||||
jersey-test-framework-grizzly2-1.9.jar
|
||||
jets3t-0.7.1.jar
|
||||
jettison-1.1.jar
|
||||
jetty-all-7.6.0.v20120127.jar
|
||||
jetty-util-6.1.26.jar
|
||||
jline-2.10.5.jar
|
||||
jline-2.12.jar
|
||||
joda-time-2.9.jar
|
||||
jodd-core-3.5.2.jar
|
||||
jpam-1.1.jar
|
||||
json-20090211.jar
|
||||
json4s-ast_2.10-3.2.10.jar
|
||||
json4s-core_2.10-3.2.10.jar
|
||||
json4s-jackson_2.10-3.2.10.jar
|
||||
jsr305-1.3.9.jar
|
||||
jta-1.1.jar
|
||||
jtransforms-2.4.0.jar
|
||||
jul-to-slf4j-1.7.10.jar
|
||||
kryo-2.21.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.2.jar
|
||||
libthrift-0.9.2.jar
|
||||
log4j-1.2.17.jar
|
||||
lz4-1.3.0.jar
|
||||
mail-1.4.1.jar
|
||||
management-api-3.0.0-b012.jar
|
||||
mesos-0.21.1-shaded-protobuf.jar
|
||||
metrics-core-3.1.2.jar
|
||||
metrics-graphite-3.1.2.jar
|
||||
metrics-json-3.1.2.jar
|
||||
metrics-jvm-3.1.2.jar
|
||||
minlog-1.2.jar
|
||||
netty-3.8.0.Final.jar
|
||||
netty-all-4.0.29.Final.jar
|
||||
objenesis-1.2.jar
|
||||
opencsv-2.3.jar
|
||||
oro-2.0.8.jar
|
||||
paranamer-2.6.jar
|
||||
parquet-column-1.7.0.jar
|
||||
parquet-common-1.7.0.jar
|
||||
parquet-encoding-1.7.0.jar
|
||||
parquet-format-2.3.0-incubating.jar
|
||||
parquet-generator-1.7.0.jar
|
||||
parquet-hadoop-1.7.0.jar
|
||||
parquet-hadoop-bundle-1.6.0.jar
|
||||
parquet-jackson-1.7.0.jar
|
||||
pmml-agent-1.2.7.jar
|
||||
pmml-model-1.2.7.jar
|
||||
pmml-schema-1.2.7.jar
|
||||
protobuf-java-2.5.0.jar
|
||||
py4j-0.9.jar
|
||||
pyrolite-4.9.jar
|
||||
quasiquotes_2.10-2.0.0-M8.jar
|
||||
reflectasm-1.07-shaded.jar
|
||||
scala-compiler-2.10.5.jar
|
||||
scala-library-2.10.5.jar
|
||||
scala-reflect-2.10.5.jar
|
||||
scalap-2.10.5.jar
|
||||
servlet-api-2.5.jar
|
||||
slf4j-api-1.7.10.jar
|
||||
slf4j-log4j12-1.7.10.jar
|
||||
snappy-0.2.jar
|
||||
snappy-java-1.1.2.jar
|
||||
spire-macros_2.10-0.7.4.jar
|
||||
spire_2.10-0.7.4.jar
|
||||
stax-api-1.0-2.jar
|
||||
stax-api-1.0.1.jar
|
||||
stream-2.7.0.jar
|
||||
stringtemplate-3.2.1.jar
|
||||
super-csv-2.2.0.jar
|
||||
tachyon-client-0.8.2.jar
|
||||
tachyon-underfs-hdfs-0.8.2.jar
|
||||
tachyon-underfs-local-0.8.2.jar
|
||||
tachyon-underfs-s3-0.8.2.jar
|
||||
uncommons-maths-1.2.2a.jar
|
||||
unused-1.0.0.jar
|
||||
xbean-asm5-shaded-4.4.jar
|
||||
xmlenc-0.52.jar
|
||||
xz-1.0.jar
|
||||
zookeeper-3.4.5.jar
|
192
dev/deps/spark-deps-hadoop-2.6
Normal file
192
dev/deps/spark-deps-hadoop-2.6
Normal file
|
@ -0,0 +1,192 @@
|
|||
JavaEWAH-0.3.2.jar
|
||||
RoaringBitmap-0.5.11.jar
|
||||
ST4-4.0.4.jar
|
||||
activation-1.1.1.jar
|
||||
akka-actor_2.10-2.3.11.jar
|
||||
akka-remote_2.10-2.3.11.jar
|
||||
akka-slf4j_2.10-2.3.11.jar
|
||||
antlr-2.7.7.jar
|
||||
antlr-runtime-3.4.jar
|
||||
aopalliance-1.0.jar
|
||||
apache-log4j-extras-1.2.17.jar
|
||||
apacheds-i18n-2.0.0-M15.jar
|
||||
apacheds-kerberos-codec-2.0.0-M15.jar
|
||||
api-asn1-api-1.0.0-M20.jar
|
||||
api-util-1.0.0-M20.jar
|
||||
arpack_combined_all-0.1.jar
|
||||
asm-3.1.jar
|
||||
asm-commons-3.1.jar
|
||||
asm-tree-3.1.jar
|
||||
avro-1.7.7.jar
|
||||
avro-ipc-1.7.7-tests.jar
|
||||
avro-ipc-1.7.7.jar
|
||||
avro-mapred-1.7.7-hadoop2.jar
|
||||
base64-2.3.8.jar
|
||||
bcprov-jdk15on-1.51.jar
|
||||
bonecp-0.8.0.RELEASE.jar
|
||||
breeze-macros_2.10-0.11.2.jar
|
||||
breeze_2.10-0.11.2.jar
|
||||
calcite-avatica-1.2.0-incubating.jar
|
||||
calcite-core-1.2.0-incubating.jar
|
||||
calcite-linq4j-1.2.0-incubating.jar
|
||||
chill-java-0.5.0.jar
|
||||
chill_2.10-0.5.0.jar
|
||||
commons-beanutils-1.7.0.jar
|
||||
commons-beanutils-core-1.8.0.jar
|
||||
commons-cli-1.2.jar
|
||||
commons-codec-1.10.jar
|
||||
commons-collections-3.2.2.jar
|
||||
commons-compiler-2.7.6.jar
|
||||
commons-compress-1.4.1.jar
|
||||
commons-configuration-1.6.jar
|
||||
commons-dbcp-1.4.jar
|
||||
commons-digester-1.8.jar
|
||||
commons-httpclient-3.1.jar
|
||||
commons-io-2.4.jar
|
||||
commons-lang-2.6.jar
|
||||
commons-lang3-3.3.2.jar
|
||||
commons-logging-1.1.3.jar
|
||||
commons-math3-3.4.1.jar
|
||||
commons-net-2.2.jar
|
||||
commons-pool-1.5.4.jar
|
||||
compress-lzf-1.0.3.jar
|
||||
config-1.2.1.jar
|
||||
core-1.1.2.jar
|
||||
curator-client-2.6.0.jar
|
||||
curator-framework-2.6.0.jar
|
||||
curator-recipes-2.6.0.jar
|
||||
datanucleus-api-jdo-3.2.6.jar
|
||||
datanucleus-core-3.2.10.jar
|
||||
datanucleus-rdbms-3.2.9.jar
|
||||
derby-10.10.1.1.jar
|
||||
eigenbase-properties-1.1.5.jar
|
||||
geronimo-annotation_1.0_spec-1.1.1.jar
|
||||
geronimo-jaspic_1.0_spec-1.0.jar
|
||||
geronimo-jta_1.1_spec-1.1.1.jar
|
||||
groovy-all-2.1.6.jar
|
||||
gson-2.2.4.jar
|
||||
guice-3.0.jar
|
||||
guice-servlet-3.0.jar
|
||||
hadoop-annotations-2.6.0.jar
|
||||
hadoop-auth-2.6.0.jar
|
||||
hadoop-client-2.6.0.jar
|
||||
hadoop-common-2.6.0.jar
|
||||
hadoop-hdfs-2.6.0.jar
|
||||
hadoop-mapreduce-client-app-2.6.0.jar
|
||||
hadoop-mapreduce-client-common-2.6.0.jar
|
||||
hadoop-mapreduce-client-core-2.6.0.jar
|
||||
hadoop-mapreduce-client-jobclient-2.6.0.jar
|
||||
hadoop-mapreduce-client-shuffle-2.6.0.jar
|
||||
hadoop-yarn-api-2.6.0.jar
|
||||
hadoop-yarn-client-2.6.0.jar
|
||||
hadoop-yarn-common-2.6.0.jar
|
||||
hadoop-yarn-server-common-2.6.0.jar
|
||||
hadoop-yarn-server-web-proxy-2.6.0.jar
|
||||
htrace-core-3.0.4.jar
|
||||
httpclient-4.3.2.jar
|
||||
httpcore-4.3.2.jar
|
||||
ivy-2.4.0.jar
|
||||
jackson-annotations-2.4.4.jar
|
||||
jackson-core-2.4.4.jar
|
||||
jackson-core-asl-1.9.13.jar
|
||||
jackson-databind-2.4.4.jar
|
||||
jackson-jaxrs-1.9.13.jar
|
||||
jackson-mapper-asl-1.9.13.jar
|
||||
jackson-module-scala_2.10-2.4.4.jar
|
||||
jackson-xc-1.9.13.jar
|
||||
janino-2.7.8.jar
|
||||
jansi-1.4.jar
|
||||
java-xmlbuilder-1.0.jar
|
||||
javax.inject-1.jar
|
||||
javax.servlet-3.0.0.v201112011016.jar
|
||||
javolution-5.5.1.jar
|
||||
jaxb-api-2.2.2.jar
|
||||
jaxb-impl-2.2.3-1.jar
|
||||
jcl-over-slf4j-1.7.10.jar
|
||||
jdo-api-3.0.1.jar
|
||||
jersey-client-1.9.jar
|
||||
jersey-core-1.9.jar
|
||||
jersey-guice-1.9.jar
|
||||
jersey-json-1.9.jar
|
||||
jersey-server-1.9.jar
|
||||
jets3t-0.9.3.jar
|
||||
jettison-1.1.jar
|
||||
jetty-6.1.26.jar
|
||||
jetty-all-7.6.0.v20120127.jar
|
||||
jetty-util-6.1.26.jar
|
||||
jline-2.10.5.jar
|
||||
jline-2.12.jar
|
||||
joda-time-2.9.jar
|
||||
jodd-core-3.5.2.jar
|
||||
jpam-1.1.jar
|
||||
json-20090211.jar
|
||||
json4s-ast_2.10-3.2.10.jar
|
||||
json4s-core_2.10-3.2.10.jar
|
||||
json4s-jackson_2.10-3.2.10.jar
|
||||
jsr305-1.3.9.jar
|
||||
jta-1.1.jar
|
||||
jtransforms-2.4.0.jar
|
||||
jul-to-slf4j-1.7.10.jar
|
||||
kryo-2.21.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.2.jar
|
||||
libthrift-0.9.2.jar
|
||||
log4j-1.2.17.jar
|
||||
lz4-1.3.0.jar
|
||||
mail-1.4.7.jar
|
||||
mesos-0.21.1-shaded-protobuf.jar
|
||||
metrics-core-3.1.2.jar
|
||||
metrics-graphite-3.1.2.jar
|
||||
metrics-json-3.1.2.jar
|
||||
metrics-jvm-3.1.2.jar
|
||||
minlog-1.2.jar
|
||||
mx4j-3.0.2.jar
|
||||
netty-3.8.0.Final.jar
|
||||
netty-all-4.0.29.Final.jar
|
||||
objenesis-1.2.jar
|
||||
opencsv-2.3.jar
|
||||
oro-2.0.8.jar
|
||||
paranamer-2.6.jar
|
||||
parquet-column-1.7.0.jar
|
||||
parquet-common-1.7.0.jar
|
||||
parquet-encoding-1.7.0.jar
|
||||
parquet-format-2.3.0-incubating.jar
|
||||
parquet-generator-1.7.0.jar
|
||||
parquet-hadoop-1.7.0.jar
|
||||
parquet-hadoop-bundle-1.6.0.jar
|
||||
parquet-jackson-1.7.0.jar
|
||||
pmml-agent-1.2.7.jar
|
||||
pmml-model-1.2.7.jar
|
||||
pmml-schema-1.2.7.jar
|
||||
protobuf-java-2.5.0.jar
|
||||
py4j-0.9.jar
|
||||
pyrolite-4.9.jar
|
||||
quasiquotes_2.10-2.0.0-M8.jar
|
||||
reflectasm-1.07-shaded.jar
|
||||
scala-compiler-2.10.5.jar
|
||||
scala-library-2.10.5.jar
|
||||
scala-reflect-2.10.5.jar
|
||||
scalap-2.10.5.jar
|
||||
servlet-api-2.5.jar
|
||||
slf4j-api-1.7.10.jar
|
||||
slf4j-log4j12-1.7.10.jar
|
||||
snappy-0.2.jar
|
||||
snappy-java-1.1.2.jar
|
||||
spire-macros_2.10-0.7.4.jar
|
||||
spire_2.10-0.7.4.jar
|
||||
stax-api-1.0-2.jar
|
||||
stax-api-1.0.1.jar
|
||||
stream-2.7.0.jar
|
||||
stringtemplate-3.2.1.jar
|
||||
super-csv-2.2.0.jar
|
||||
tachyon-client-0.8.2.jar
|
||||
tachyon-underfs-hdfs-0.8.2.jar
|
||||
tachyon-underfs-local-0.8.2.jar
|
||||
tachyon-underfs-s3-0.8.2.jar
|
||||
uncommons-maths-1.2.2a.jar
|
||||
unused-1.0.0.jar
|
||||
xbean-asm5-shaded-4.4.jar
|
||||
xercesImpl-2.9.1.jar
|
||||
xmlenc-0.52.jar
|
||||
xz-1.0.jar
|
||||
zookeeper-3.4.6.jar
|
|
@ -168,6 +168,10 @@ def main():
|
|||
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.2"
|
||||
if "test-hadoop2.3" in ghprb_pull_title:
|
||||
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.3"
|
||||
if "test-hadoop2.4" in ghprb_pull_title:
|
||||
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.4"
|
||||
if "test-hadoop2.6" in ghprb_pull_title:
|
||||
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.6"
|
||||
|
||||
build_display_name = os.environ["BUILD_DISPLAY_NAME"]
|
||||
build_url = os.environ["BUILD_URL"]
|
||||
|
|
|
@ -295,13 +295,14 @@ def exec_sbt(sbt_args=()):
|
|||
|
||||
def get_hadoop_profiles(hadoop_version):
|
||||
"""
|
||||
For the given Hadoop version tag, return a list of SBT profile flags for
|
||||
For the given Hadoop version tag, return a list of Maven/SBT profile flags for
|
||||
building and testing against that Hadoop version.
|
||||
"""
|
||||
|
||||
sbt_maven_hadoop_profiles = {
|
||||
"hadoop2.2": ["-Pyarn", "-Phadoop-2.2"],
|
||||
"hadoop2.3": ["-Pyarn", "-Phadoop-2.3", "-Dhadoop.version=2.3.0"],
|
||||
"hadoop2.3": ["-Pyarn", "-Phadoop-2.3"],
|
||||
"hadoop2.4": ["-Pyarn", "-Phadoop-2.4"],
|
||||
"hadoop2.6": ["-Pyarn", "-Phadoop-2.6"],
|
||||
}
|
||||
|
||||
|
|
|
@ -32,8 +32,10 @@ export LC_ALL=C
|
|||
HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pyarn -Phive"
|
||||
MVN="build/mvn --force"
|
||||
HADOOP_PROFILES=(
|
||||
hadoop-2.2
|
||||
hadoop-2.3
|
||||
hadoop-2.4
|
||||
hadoop-2.6
|
||||
)
|
||||
|
||||
# We'll switch the version to a temp. one, publish POMs using that new version, then switch back to
|
||||
|
|
Loading…
Reference in a new issue