ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-36759][BUILD] Upgrade Scala to 2.12.15

Browse source 
### What changes were proposed in this pull request?

This PR aims to upgrade Scala to 2.12.15 to support Java 17/18 better.

### Why are the changes needed?

Scala 2.12.15 improves compatibility with JDK 17 and 18:

https://github.com/scala/scala/releases/tag/v2.12.15

- Avoids IllegalArgumentException in JDK 17+ for lambda deserialization
- Upgrades to ASM 9.2, for JDK 18 support in optimizer

### Does this PR introduce _any_ user-facing change?

Yes, this is a Scala version change.

### How was this patch tested?

Pass the CIs

Closes #33999 from dongjoon-hyun/SPARK-36759.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Dongjoon Hyun 2021-09-15 13:43:25 -07:00
parent a927b0836b
commit 16f1f71ba5
4 changed files with 9 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
dev/deps/spark-deps-hadoop-2.7-hive-2.3
View file

@ -212,10 +212,10 @@ py4j/0.10.9.2//py4j-0.10.9.2.jar
pyrolite/4.30//pyrolite-4.30.jar pyrolite/4.30//pyrolite-4.30.jar
rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar
scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
scala-compiler/2.12.14//scala-compiler-2.12.14.jar scala-compiler/2.12.15//scala-compiler-2.12.15.jar
scala-library/2.12.14//scala-library-2.12.14.jar scala-library/2.12.15//scala-library-2.12.15.jar
scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
scala-reflect/2.12.14//scala-reflect-2.12.14.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar
scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
shims/0.9.0//shims-0.9.0.jar shims/0.9.0//shims-0.9.0.jar

6
dev/deps/spark-deps-hadoop-3.2-hive-2.3
View file

@ -183,10 +183,10 @@ py4j/0.10.9.2//py4j-0.10.9.2.jar
pyrolite/4.30//pyrolite-4.30.jar pyrolite/4.30//pyrolite-4.30.jar
rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar rocksdbjni/6.20.3//rocksdbjni-6.20.3.jar
scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
scala-compiler/2.12.14//scala-compiler-2.12.14.jar scala-compiler/2.12.15//scala-compiler-2.12.15.jar
scala-library/2.12.14//scala-library-2.12.14.jar scala-library/2.12.15//scala-library-2.12.15.jar
scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
scala-reflect/2.12.14//scala-reflect-2.12.14.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar
scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
shims/0.9.0//shims-0.9.0.jar shims/0.9.0//shims-0.9.0.jar

4
pom.xml
View file

@ -160,7 +160,7 @@
<commons.math3.version>3.4.1</commons.math3.version> <commons.math3.version>3.4.1</commons.math3.version>
<!-- managed up from 3.2.1 for SPARK-11652 --> <!-- managed up from 3.2.1 for SPARK-11652 -->
<commons.collections.version>3.2.2</commons.collections.version> <commons.collections.version>3.2.2</commons.collections.version>
<scala.version>2.12.14</scala.version> <scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version> <scala.binary.version>2.12</scala.binary.version>
<scalatest-maven-plugin.version>2.0.2</scalatest-maven-plugin.version> <scalatest-maven-plugin.version>2.0.2</scalatest-maven-plugin.version>
<scalafmt.parameters>--test</scalafmt.parameters> <scalafmt.parameters>--test</scalafmt.parameters>
@ -2648,7 +2648,7 @@
<compilerPlugin> <compilerPlugin>
<groupId>com.github.ghik</groupId> <groupId>com.github.ghik</groupId>
<artifactId>silencer-plugin_${scala.version}</artifactId> <artifactId>silencer-plugin_${scala.version}</artifactId>
<version>1.7.5</version> <version>1.7.6</version>
</compilerPlugin> </compilerPlugin>
</compilerPlugins> </compilerPlugins>
</configuration> </configuration>

2
project/SparkBuild.scala
View file

@ -208,7 +208,7 @@ object SparkBuild extends PomBuild {
lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq( lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq(
libraryDependencies ++= { libraryDependencies ++= {
if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) { if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) {
val silencerVersion = "1.7.5" val silencerVersion = "1.7.6"
Seq( Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0", "org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0",
compilerPlugin("com.github.ghik" % "silencer-plugin" % silencerVersion cross CrossVersion.full), compilerPlugin("com.github.ghik" % "silencer-plugin" % silencerVersion cross CrossVersion.full),