[SPARK-23063][K8S] K8s changes for publishing scripts (and a couple of other misses)
## What changes were proposed in this pull request? Including the `-Pkubernetes` flag in a few places it was missed. ## How was this patch tested? checkstyle, mima through manual tests. Author: foxish <ramanathana@google.com> Closes #20256 from foxish/SPARK-23063.
This commit is contained in:
parent
afae8f2bc8
commit
c3548d11c3
|
@ -92,9 +92,9 @@ MVN="build/mvn --force"
|
|||
# Hive-specific profiles for some builds
|
||||
HIVE_PROFILES="-Phive -Phive-thriftserver"
|
||||
# Profiles for publishing snapshots and release to Maven Central
|
||||
PUBLISH_PROFILES="-Pmesos -Pyarn -Pflume $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
|
||||
PUBLISH_PROFILES="-Pmesos -Pyarn -Pkubernetes -Pflume $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl"
|
||||
# Profiles for building binary releases
|
||||
BASE_RELEASE_PROFILES="-Pmesos -Pyarn -Pflume -Psparkr"
|
||||
BASE_RELEASE_PROFILES="-Pmesos -Pyarn -Pkubernetes -Pflume -Psparkr"
|
||||
# Scala 2.11 only profiles for some builds
|
||||
SCALA_2_11_PROFILES="-Pkafka-0-8"
|
||||
# Scala 2.12 only profiles for some builds
|
||||
|
|
|
@ -185,6 +185,8 @@ known_components = {
|
|||
"graphx": "GraphX",
|
||||
"input/output": CORE_COMPONENT,
|
||||
"java api": "Java API",
|
||||
"k8s": "Kubernetes",
|
||||
"kubernetes": "Kubernetes",
|
||||
"mesos": "Mesos",
|
||||
"ml": "MLlib",
|
||||
"mllib": "MLlib",
|
||||
|
|
|
@ -17,6 +17,7 @@ arpack_combined_all-0.1.jar
|
|||
arrow-format-0.8.0.jar
|
||||
arrow-memory-0.8.0.jar
|
||||
arrow-vector-0.8.0.jar
|
||||
automaton-1.11-8.jar
|
||||
avro-1.7.7.jar
|
||||
avro-ipc-1.7.7.jar
|
||||
avro-mapred-1.7.7-hadoop2.jar
|
||||
|
@ -60,6 +61,7 @@ datanucleus-rdbms-3.2.9.jar
|
|||
derby-10.12.1.1.jar
|
||||
eigenbase-properties-1.1.5.jar
|
||||
flatbuffers-1.2.0-3f79e055.jar
|
||||
generex-1.0.1.jar
|
||||
gson-2.2.4.jar
|
||||
guava-14.0.1.jar
|
||||
guice-3.0.jar
|
||||
|
@ -91,8 +93,10 @@ jackson-annotations-2.6.7.jar
|
|||
jackson-core-2.6.7.jar
|
||||
jackson-core-asl-1.9.13.jar
|
||||
jackson-databind-2.6.7.1.jar
|
||||
jackson-dataformat-yaml-2.6.7.jar
|
||||
jackson-jaxrs-1.9.13.jar
|
||||
jackson-mapper-asl-1.9.13.jar
|
||||
jackson-module-jaxb-annotations-2.6.7.jar
|
||||
jackson-module-paranamer-2.7.9.jar
|
||||
jackson-module-scala_2.11-2.6.7.1.jar
|
||||
jackson-xc-1.9.13.jar
|
||||
|
@ -131,10 +135,13 @@ jta-1.1.jar
|
|||
jtransforms-2.4.0.jar
|
||||
jul-to-slf4j-1.7.16.jar
|
||||
kryo-shaded-3.0.3.jar
|
||||
kubernetes-client-3.0.0.jar
|
||||
kubernetes-model-2.0.0.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.3.jar
|
||||
libthrift-0.9.3.jar
|
||||
log4j-1.2.17.jar
|
||||
logging-interceptor-3.8.1.jar
|
||||
lz4-java-1.4.0.jar
|
||||
machinist_2.11-0.6.1.jar
|
||||
macro-compat_2.11-1.1.1.jar
|
||||
|
@ -147,6 +154,8 @@ minlog-1.3.0.jar
|
|||
netty-3.9.9.Final.jar
|
||||
netty-all-4.1.17.Final.jar
|
||||
objenesis-2.1.jar
|
||||
okhttp-3.8.1.jar
|
||||
okio-1.13.0.jar
|
||||
opencsv-2.3.jar
|
||||
orc-core-1.4.1-nohive.jar
|
||||
orc-mapreduce-1.4.1-nohive.jar
|
||||
|
@ -171,6 +180,7 @@ scala-xml_2.11-1.0.5.jar
|
|||
shapeless_2.11-2.3.2.jar
|
||||
slf4j-api-1.7.16.jar
|
||||
slf4j-log4j12-1.7.16.jar
|
||||
snakeyaml-1.15.jar
|
||||
snappy-0.2.jar
|
||||
snappy-java-1.1.2.6.jar
|
||||
spire-macros_2.11-0.13.0.jar
|
||||
|
@ -186,5 +196,6 @@ xbean-asm5-shaded-4.4.jar
|
|||
xercesImpl-2.9.1.jar
|
||||
xmlenc-0.52.jar
|
||||
xz-1.0.jar
|
||||
zjsonpatch-0.3.0.jar
|
||||
zookeeper-3.4.6.jar
|
||||
zstd-jni-1.3.2-2.jar
|
||||
|
|
|
@ -17,6 +17,7 @@ arpack_combined_all-0.1.jar
|
|||
arrow-format-0.8.0.jar
|
||||
arrow-memory-0.8.0.jar
|
||||
arrow-vector-0.8.0.jar
|
||||
automaton-1.11-8.jar
|
||||
avro-1.7.7.jar
|
||||
avro-ipc-1.7.7.jar
|
||||
avro-mapred-1.7.7-hadoop2.jar
|
||||
|
@ -60,6 +61,7 @@ datanucleus-rdbms-3.2.9.jar
|
|||
derby-10.12.1.1.jar
|
||||
eigenbase-properties-1.1.5.jar
|
||||
flatbuffers-1.2.0-3f79e055.jar
|
||||
generex-1.0.1.jar
|
||||
gson-2.2.4.jar
|
||||
guava-14.0.1.jar
|
||||
guice-3.0.jar
|
||||
|
@ -91,8 +93,10 @@ jackson-annotations-2.6.7.jar
|
|||
jackson-core-2.6.7.jar
|
||||
jackson-core-asl-1.9.13.jar
|
||||
jackson-databind-2.6.7.1.jar
|
||||
jackson-dataformat-yaml-2.6.7.jar
|
||||
jackson-jaxrs-1.9.13.jar
|
||||
jackson-mapper-asl-1.9.13.jar
|
||||
jackson-module-jaxb-annotations-2.6.7.jar
|
||||
jackson-module-paranamer-2.7.9.jar
|
||||
jackson-module-scala_2.11-2.6.7.1.jar
|
||||
jackson-xc-1.9.13.jar
|
||||
|
@ -132,10 +136,13 @@ jta-1.1.jar
|
|||
jtransforms-2.4.0.jar
|
||||
jul-to-slf4j-1.7.16.jar
|
||||
kryo-shaded-3.0.3.jar
|
||||
kubernetes-client-3.0.0.jar
|
||||
kubernetes-model-2.0.0.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.3.jar
|
||||
libthrift-0.9.3.jar
|
||||
log4j-1.2.17.jar
|
||||
logging-interceptor-3.8.1.jar
|
||||
lz4-java-1.4.0.jar
|
||||
machinist_2.11-0.6.1.jar
|
||||
macro-compat_2.11-1.1.1.jar
|
||||
|
@ -148,6 +155,8 @@ minlog-1.3.0.jar
|
|||
netty-3.9.9.Final.jar
|
||||
netty-all-4.1.17.Final.jar
|
||||
objenesis-2.1.jar
|
||||
okhttp-3.8.1.jar
|
||||
okio-1.13.0.jar
|
||||
opencsv-2.3.jar
|
||||
orc-core-1.4.1-nohive.jar
|
||||
orc-mapreduce-1.4.1-nohive.jar
|
||||
|
@ -172,6 +181,7 @@ scala-xml_2.11-1.0.5.jar
|
|||
shapeless_2.11-2.3.2.jar
|
||||
slf4j-api-1.7.16.jar
|
||||
slf4j-log4j12-1.7.16.jar
|
||||
snakeyaml-1.15.jar
|
||||
snappy-0.2.jar
|
||||
snappy-java-1.1.2.6.jar
|
||||
spire-macros_2.11-0.13.0.jar
|
||||
|
@ -187,5 +197,6 @@ xbean-asm5-shaded-4.4.jar
|
|||
xercesImpl-2.9.1.jar
|
||||
xmlenc-0.52.jar
|
||||
xz-1.0.jar
|
||||
zjsonpatch-0.3.0.jar
|
||||
zookeeper-3.4.6.jar
|
||||
zstd-jni-1.3.2-2.jar
|
||||
|
|
|
@ -20,7 +20,7 @@
|
|||
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
|
||||
SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
|
||||
|
||||
ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pmesos -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR)
|
||||
ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR)
|
||||
|
||||
if test ! -z "$ERRORS"; then
|
||||
echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS"
|
||||
|
|
2
dev/mima
2
dev/mima
|
@ -24,7 +24,7 @@ set -e
|
|||
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
|
||||
cd "$FWDIR"
|
||||
|
||||
SPARK_PROFILES="-Pmesos -Pkafka-0-8 -Pyarn -Pflume -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
|
||||
SPARK_PROFILES="-Pmesos -Pkafka-0-8 -Pkubernetes -Pyarn -Pflume -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
|
||||
TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
|
||||
OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
|
||||
|
||||
|
|
|
@ -24,6 +24,7 @@ ERRORS=$(echo -e "q\n" \
|
|||
-Pkinesis-asl \
|
||||
-Pmesos \
|
||||
-Pkafka-0-8 \
|
||||
-Pkubernetes \
|
||||
-Pyarn \
|
||||
-Pflume \
|
||||
-Phive \
|
||||
|
|
|
@ -29,7 +29,7 @@ export LC_ALL=C
|
|||
# TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
|
||||
|
||||
# NOTE: These should match those in the release publishing script
|
||||
HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkafka-0-8 -Pyarn -Pflume -Phive"
|
||||
HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkafka-0-8 -Pkubernetes -Pyarn -Pflume -Phive"
|
||||
MVN="build/mvn"
|
||||
HADOOP_PROFILES=(
|
||||
hadoop-2.6
|
||||
|
|
Loading…
Reference in a new issue