ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-25956] Make Scala 2.12 as default Scala version in Spark 3.0

Browse source 
## What changes were proposed in this pull request?

This PR makes Spark's default Scala version as 2.12, and Scala 2.11 will be the alternative version. This implies that Scala 2.12 will be used by our CI builds including pull request builds.

We'll update the Jenkins to include a new compile-only jobs for Scala 2.11 to ensure the code can be still compiled with Scala 2.11.

## How was this patch tested?

existing tests

Closes #22967 from dbtsai/scala2.12.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
DB Tsai 2018-11-14 16:22:23 -08:00 committed by Dongjoon Hyun
parent 2977e2312d
commit ad853c5678
No known key found for this signature in database
GPG key ID: EDA00CE834F0FC5C
45 changed files with 138 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
assembly/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-assembly_2.11</artifactId>
<artifactId>spark-assembly_2.12</artifactId>
<name>Spark Project Assembly</name>
<url>http://spark.apache.org/</url>
<packaging>pom</packaging>

4
common/kvstore/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-kvstore_2.11</artifactId>
<artifactId>spark-kvstore_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Local DB</name>
<url>http://spark.apache.org/</url>

4
common/network-common/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-network-common_2.11</artifactId>
<artifactId>spark-network-common_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Networking</name>
<url>http://spark.apache.org/</url>

4
common/network-shuffle/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-network-shuffle_2.11</artifactId>
<artifactId>spark-network-shuffle_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Shuffle Streaming Service</name>
<url>http://spark.apache.org/</url>

4
common/network-yarn/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-network-yarn_2.11</artifactId>
<artifactId>spark-network-yarn_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN Shuffle Service</name>
<url>http://spark.apache.org/</url>

4
common/sketch/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-sketch_2.11</artifactId>
<artifactId>spark-sketch_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Sketch</name>
<url>http://spark.apache.org/</url>

4
common/tags/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-tags_2.11</artifactId>
<artifactId>spark-tags_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Tags</name>
<url>http://spark.apache.org/</url>

4
common/unsafe/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-unsafe_2.11</artifactId>
<artifactId>spark-unsafe_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Unsafe</name>
<url>http://spark.apache.org/</url>

4
core/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
<properties>
<sbt.project.name>core</sbt.project.name>
</properties>

36
dev/deps/spark-deps-hadoop-2.7
View file

@ -22,13 +22,13 @@ avro-1.8.2.jar
avro-ipc-1.8.2.jar
avro-mapred-1.8.2-hadoop2.jar
bonecp-0.8.0.RELEASE.jar
breeze-macros_2.11-0.13.2.jar
breeze_2.11-0.13.2.jar
breeze-macros_2.12-0.13.2.jar
breeze_2.12-0.13.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.9.3.jar
chill_2.11-0.9.3.jar
chill_2.12-0.9.3.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@ -96,7 +96,7 @@ jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
jackson-module-jaxb-annotations-2.9.6.jar
jackson-module-paranamer-2.9.6.jar
jackson-module-scala_2.11-2.9.6.jar
jackson-module-scala_2.12-2.9.6.jar
jackson-xc-1.9.13.jar
janino-3.0.10.jar
javassist-3.18.1-GA.jar
@ -122,10 +122,10 @@ jline-2.14.6.jar
joda-time-2.9.3.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json4s-ast_2.11-3.5.3.jar
json4s-core_2.11-3.5.3.jar
json4s-jackson_2.11-3.5.3.jar
json4s-scalap_2.11-3.5.3.jar
json4s-ast_2.12-3.5.3.jar
json4s-core_2.12-3.5.3.jar
json4s-jackson_2.12-3.5.3.jar
json4s-scalap_2.12-3.5.3.jar
jsp-api-2.1.jar
jsr305-3.0.0.jar
jta-1.1.jar
@ -140,8 +140,8 @@ libthrift-0.9.3.jar
log4j-1.2.17.jar
logging-interceptor-3.9.1.jar
lz4-java-1.5.0.jar
machinist_2.11-0.6.1.jar
macro-compat_2.11-1.1.1.jar
machinist_2.12-0.6.1.jar
macro-compat_2.12-1.1.1.jar
mesos-1.4.0-shaded-protobuf.jar
metrics-core-3.1.5.jar
metrics-graphite-3.1.5.jar
@ -170,19 +170,19 @@ parquet-jackson-1.10.0.jar
protobuf-java-2.5.0.jar
py4j-0.10.8.1.jar
pyrolite-4.13.jar
scala-compiler-2.11.12.jar
scala-library-2.11.12.jar
scala-parser-combinators_2.11-1.1.0.jar
scala-reflect-2.11.12.jar
scala-xml_2.11-1.0.5.jar
shapeless_2.11-2.3.2.jar
scala-compiler-2.12.7.jar
scala-library-2.12.7.jar
scala-parser-combinators_2.12-1.1.0.jar
scala-reflect-2.12.7.jar
scala-xml_2.12-1.0.5.jar
shapeless_2.12-2.3.2.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
snakeyaml-1.18.jar
snappy-0.2.jar
snappy-java-1.1.7.1.jar
spire-macros_2.11-0.13.0.jar
spire_2.11-0.13.0.jar
spire-macros_2.12-0.13.0.jar
spire_2.12-0.13.0.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar

36
dev/deps/spark-deps-hadoop-3.1
View file

@ -20,13 +20,13 @@ avro-1.8.2.jar
avro-ipc-1.8.2.jar
avro-mapred-1.8.2-hadoop2.jar
bonecp-0.8.0.RELEASE.jar
breeze-macros_2.11-0.13.2.jar
breeze_2.11-0.13.2.jar
breeze-macros_2.12-0.13.2.jar
breeze_2.12-0.13.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.9.3.jar
chill_2.11-0.9.3.jar
chill_2.12-0.9.3.jar
commons-beanutils-1.9.3.jar
commons-cli-1.2.jar
commons-codec-1.10.jar
@ -96,7 +96,7 @@ jackson-jaxrs-json-provider-2.7.8.jar
jackson-mapper-asl-1.9.13.jar
jackson-module-jaxb-annotations-2.9.6.jar
jackson-module-paranamer-2.9.6.jar
jackson-module-scala_2.11-2.9.6.jar
jackson-module-scala_2.12-2.9.6.jar
janino-3.0.10.jar
javassist-3.18.1-GA.jar
javax.annotation-api-1.2.jar
@ -123,10 +123,10 @@ joda-time-2.9.3.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-smart-2.3.jar
json4s-ast_2.11-3.5.3.jar
json4s-core_2.11-3.5.3.jar
json4s-jackson_2.11-3.5.3.jar
json4s-scalap_2.11-3.5.3.jar
json4s-ast_2.12-3.5.3.jar
json4s-core_2.12-3.5.3.jar
json4s-jackson_2.12-3.5.3.jar
json4s-scalap_2.12-3.5.3.jar
jsp-api-2.1.jar
jsr305-3.0.0.jar
jta-1.1.jar
@ -155,8 +155,8 @@ libthrift-0.9.3.jar
log4j-1.2.17.jar
logging-interceptor-3.9.1.jar
lz4-java-1.5.0.jar
machinist_2.11-0.6.1.jar
macro-compat_2.11-1.1.1.jar
machinist_2.12-0.6.1.jar
macro-compat_2.12-1.1.1.jar
mesos-1.4.0-shaded-protobuf.jar
metrics-core-3.1.5.jar
metrics-graphite-3.1.5.jar
@ -189,19 +189,19 @@ protobuf-java-2.5.0.jar
py4j-0.10.8.1.jar
pyrolite-4.13.jar
re2j-1.1.jar
scala-compiler-2.11.12.jar
scala-library-2.11.12.jar
scala-parser-combinators_2.11-1.1.0.jar
scala-reflect-2.11.12.jar
scala-xml_2.11-1.0.5.jar
shapeless_2.11-2.3.2.jar
scala-compiler-2.12.7.jar
scala-library-2.12.7.jar
scala-parser-combinators_2.12-1.1.0.jar
scala-reflect-2.12.7.jar
scala-xml_2.12-1.0.5.jar
shapeless_2.12-2.3.2.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
snakeyaml-1.18.jar
snappy-0.2.jar
snappy-java-1.1.7.1.jar
spire-macros_2.11-0.13.0.jar
spire_2.11-0.13.0.jar
spire-macros_2.12-0.13.0.jar
spire_2.12-0.13.0.jar
stax-api-1.0.1.jar
stax2-api-3.1.4.jar
stream-2.7.0.jar

4
docs/_config.yml
View file

@ -16,8 +16,8 @@ include:
# of Spark, Scala, and Mesos.
SPARK_VERSION: 3.0.0-SNAPSHOT
SPARK_VERSION_SHORT: 3.0.0
SCALA_BINARY_VERSION: "2.11"
SCALA_VERSION: "2.11.12"
SCALA_BINARY_VERSION: "2.12"
SCALA_VERSION: "2.12.7"
MESOS_VERSION: 1.0.0
SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK
SPARK_GITHUB_URL: https://github.com/apache/spark

2
docs/_plugins/copy_api_dirs.rb
View file

@ -37,7 +37,7 @@ if not (ENV['SKIP_API'] == '1')
# Copy over the unified ScalaDoc for all projects to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
source = "../target/scala-2.11/unidoc"
source = "../target/scala-2.12/unidoc"
dest = "api/scala"
puts "Making directory " + dest

18
docs/building-spark.md
View file

@ -96,9 +96,9 @@ It's possible to build Spark submodules using the `mvn -pl` option.
For instance, you can build the Spark Streaming module using:
./build/mvn -pl :spark-streaming_2.11 clean install
./build/mvn -pl :spark-streaming_{{site.SCALA_BINARY_VERSION}} clean install
where `spark-streaming_2.11` is the `artifactId` as defined in `streaming/pom.xml` file.
where `spark-streaming_{{site.SCALA_BINARY_VERSION}}` is the `artifactId` as defined in `streaming/pom.xml` file.
## Continuous Compilation
@ -230,7 +230,7 @@ Once installed, the `docker` service needs to be started, if not already running
On Linux, this can be done by `sudo service docker start`.
./build/mvn install -DskipTests
./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.11
./build/mvn test -Pdocker-integration-tests -pl :spark-docker-integration-tests_{{site.SCALA_BINARY_VERSION}}
or
@ -238,17 +238,17 @@ or
## Change Scala Version
To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.12):
To build Spark using another supported Scala version, please change the major Scala version using (e.g. 2.11):
./dev/change-scala-version.sh 2.12
./dev/change-scala-version.sh 2.11
For Maven, please enable the profile (e.g. 2.12):
For Maven, please enable the profile (e.g. 2.11):
./build/mvn -Pscala-2.12 compile
./build/mvn -Pscala-2.11 compile
For SBT, specify a complete scala version using (e.g. 2.12.6):
For SBT, specify a complete scala version using (e.g. 2.11.12):
./build/sbt -Dscala.version=2.12.6
./build/sbt -Dscala.version=2.11.12
Otherwise, the sbt-pom-reader plugin will use the `scala.version` specified in the spark-parent pom.

2
docs/cloud-integration.md
View file

@ -85,7 +85,7 @@ is set to the chosen version of Spark:
...
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>hadoop-cloud_2.11</artifactId>
<artifactId>hadoop-cloud_{{site.SCALA_BINARY_VERSION}}</artifactId>
<version>${spark.version}</version>
</dependency>
...

2
docs/sparkr.md
View file

@ -133,7 +133,7 @@ specifying `--packages` with `spark-submit` or `sparkR` commands, or if initiali
<div data-lang="r" markdown="1">
{% highlight r %}
sparkR.session(sparkPackages = "com.databricks:spark-avro_2.11:3.0.0")
sparkR.session(sparkPackages = "org.apache.spark:spark-avro_{{site.SCALA_BINARY_VERSION}}:{{site.SPARK_VERSION}}")
{% endhighlight %}
</div>

4
examples/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-examples_2.11</artifactId>
<artifactId>spark-examples_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Examples</name>
<url>http://spark.apache.org/</url>

4
external/avro/pom.xml vendored
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
<properties>
<sbt.project.name>avro</sbt.project.name>
</properties>

4
external/docker-integration-tests/pom.xml vendored
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-docker-integration-tests_2.11</artifactId>
<artifactId>spark-docker-integration-tests_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Docker Integration Tests</name>
<url>http://spark.apache.org/</url>

4
external/kafka-0-10-assembly/pom.xml vendored
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-streaming-kafka-0-10-assembly_2.11</artifactId>
<artifactId>spark-streaming-kafka-0-10-assembly_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Integration for Kafka 0.10 Assembly</name>
<url>http://spark.apache.org/</url>

4
external/kafka-0-10-sql/pom.xml vendored
View file

@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-kafka-0-10_2.11</artifactId>
<artifactId>spark-sql-kafka-0-10_2.12</artifactId>
<properties>
<sbt.project.name>sql-kafka-0-10</sbt.project.name>
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->

4
external/kafka-0-10/pom.xml vendored
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
<properties>
<sbt.project.name>streaming-kafka-0-10</sbt.project.name>
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->

4
external/kinesis-asl-assembly/pom.xml vendored
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-streaming-kinesis-asl-assembly_2.11</artifactId>
<artifactId>spark-streaming-kinesis-asl-assembly_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Kinesis Assembly</name>
<url>http://spark.apache.org/</url>

4
external/kinesis-asl/pom.xml vendored
View file

@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Kinesis integration is not included by default due to ASL-licensed code. -->
<artifactId>spark-streaming-kinesis-asl_2.11</artifactId>
<artifactId>spark-streaming-kinesis-asl_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Kinesis Integration</name>

4
external/spark-ganglia-lgpl/pom.xml vendored
View file

@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
<artifactId>spark-ganglia-lgpl_2.11</artifactId>
<artifactId>spark-ganglia-lgpl_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Ganglia Integration</name>

4
graphx/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-graphx_2.11</artifactId>
<artifactId>spark-graphx_2.12</artifactId>
<properties>
<sbt.project.name>graphx</sbt.project.name>
</properties>

4
hadoop-cloud/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-hadoop-cloud_2.11</artifactId>
<artifactId>spark-hadoop-cloud_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Cloud Integration through Hadoop Libraries</name>
<description>

4
launcher/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-launcher_2.11</artifactId>
<artifactId>spark-launcher_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Launcher</name>
<url>http://spark.apache.org/</url>

4
mllib-local/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-mllib-local_2.11</artifactId>
<artifactId>spark-mllib-local_2.12</artifactId>
<properties>
<sbt.project.name>mllib-local</sbt.project.name>
</properties>

4
mllib/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-mllib_2.11</artifactId>
<artifactId>spark-mllib_2.12</artifactId>
<properties>
<sbt.project.name>mllib</sbt.project.name>
</properties>

20
pom.xml
View file

@ -25,7 +25,7 @@
<version>18</version>
</parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Spark Project Parent POM</name>
@ -154,8 +154,8 @@
<commons.math3.version>3.4.1</commons.math3.version>
<!-- managed up from 3.2.1 for SPARK-11652 -->
<commons.collections.version>3.2.2</commons.collections.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.12.7</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<fasterxml.jackson.version>2.9.6</fasterxml.jackson.version>
<snappy.version>1.1.7.1</snappy.version>
@ -1998,6 +1998,7 @@
-->
<exclude>org.jboss.netty</exclude>
<exclude>org.codehaus.groovy</exclude>
<exclude>*:*_2.11</exclude>
<exclude>*:*_2.10</exclude>
</excludes>
<searchTransitive>true</searchTransitive>
@ -2705,14 +2706,14 @@
<!-- Exists for backwards compatibility; profile doesn't do anything -->
<profile>
<id>scala-2.11</id>
<id>scala-2.12</id>
</profile>
<profile>
<id>scala-2.12</id>
<id>scala-2.11</id>
<properties>
<scala.version>2.12.7</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
</properties>
<build>
<plugins>
@ -2728,8 +2729,9 @@
<configuration>
<rules>
<bannedDependencies>
<excludes combine.children="append">
<exclude>*:*_2.11</exclude>
<excludes combine.self="override">
<exclude>org.jboss.netty</exclude>
<exclude>org.codehaus.groovy</exclude>
<exclude>*:*_2.10</exclude>
</excludes>
</bannedDependencies>

2
project/MimaBuild.scala
View file

@ -90,7 +90,7 @@ object MimaBuild {
val organization = "org.apache.spark"
val previousSparkVersion = "2.4.0"
val project = projectRef.project
val fullId = "spark-" + project + "_2.11"
val fullId = "spark-" + project + "_2.12"
mimaDefaultSettings ++
Seq(mimaPreviousArtifacts := Set(organization % fullId % previousSparkVersion),
mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value))

14
project/SparkBuild.scala
View file

@ -95,15 +95,15 @@ object SparkBuild extends PomBuild {
}
Option(System.getProperty("scala.version"))
.filter(_.startsWith("2.12"))
.filter(_.startsWith("2.11"))
.foreach { versionString =>
System.setProperty("scala-2.12", "true")
System.setProperty("scala-2.11", "true")
}
if (System.getProperty("scala-2.12") == "") {
if (System.getProperty("scala-2.11") == "") {
// To activate scala-2.10 profile, replace empty property value to non-empty value
// in the same way as Maven which handles -Dname as -Dname=true before executes build process.
// see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082
System.setProperty("scala-2.12", "true")
System.setProperty("scala-2.11", "true")
}
profiles
}
@ -849,10 +849,10 @@ object TestSettings {
import BuildCommons._
private val scalaBinaryVersion =
if (System.getProperty("scala-2.12") == "true") {
"2.12"
} else {
if (System.getProperty("scala-2.11") == "true") {
"2.11"
} else {
"2.12"
}
lazy val settings = Seq (
// Fork new JVMs for tests and set Java options for those

4
python/run-tests.py
View file

@ -59,9 +59,7 @@ FAILURE_REPORTING_LOCK = Lock()
LOGGER = logging.getLogger()
# Find out where the assembly jars are located.
# Later, add back 2.12 to this list:
# for scala in ["2.11", "2.12"]:
for scala in ["2.11"]:
for scala in ["2.11", "2.12"]:
build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala)
if os.path.isdir(build_dir):
SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*")

4
repl/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-repl_2.11</artifactId>
<artifactId>spark-repl_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project REPL</name>
<url>http://spark.apache.org/</url>

4
resource-managers/kubernetes/core/pom.xml
View file

@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>spark-kubernetes_2.11</artifactId>
<artifactId>spark-kubernetes_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Kubernetes</name>
<properties>

4
resource-managers/kubernetes/integration-tests/pom.xml
View file

@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>spark-kubernetes-integration-tests_2.11</artifactId>
<artifactId>spark-kubernetes-integration-tests_2.12</artifactId>
<properties>
<download-maven-plugin.version>1.3.0</download-maven-plugin.version>
<exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>

4
resource-managers/mesos/pom.xml
View file

@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-mesos_2.11</artifactId>
<artifactId>spark-mesos_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Mesos</name>
<properties>

4
resource-managers/yarn/pom.xml
View file

@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-yarn_2.11</artifactId>
<artifactId>spark-yarn_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN</name>
<properties>

4
sql/catalyst/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-catalyst_2.11</artifactId>
<artifactId>spark-catalyst_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Catalyst</name>
<url>http://spark.apache.org/</url>

4
sql/core/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project SQL</name>
<url>http://spark.apache.org/</url>

4
sql/hive-thriftserver/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-hive-thriftserver_2.11</artifactId>
<artifactId>spark-hive-thriftserver_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive Thrift Server</name>
<url>http://spark.apache.org/</url>

4
sql/hive/pom.xml
View file

@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-hive_2.11</artifactId>
<artifactId>spark-hive_2.12</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive</name>
<url>http://spark.apache.org/</url>

4
streaming/pom.xml
View file

@ -20,12 +20,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-streaming_2.11</artifactId>
<artifactId>spark-streaming_2.12</artifactId>
<properties>
<sbt.project.name>streaming</sbt.project.name>
</properties>

4
tools/pom.xml
View file

@ -19,12 +19,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<artifactId>spark-parent_2.12</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>spark-tools_2.11</artifactId>
<artifactId>spark-tools_2.12</artifactId>
<properties>
<sbt.project.name>tools</sbt.project.name>
</properties>