[SPARK-36712][BUILD] Make scala-parallel-collections in 2.13 POM a direct dependency (not in maven profile)
As [reported on `devspark.apache.org`](https://lists.apache.org/thread.html/r84cff66217de438f1389899e6d6891b573780159cd45463acf3657aa%40%3Cdev.spark.apache.org%3E), the published POMs when building with Scala 2.13 have the `scala-parallel-collections` dependency only in the `scala-2.13` profile of the pom. ### What changes were proposed in this pull request? This PR suggests to work around this by un-commenting the `scala-parallel-collections` dependency when switching to 2.13 using the the `change-scala-version.sh` script. I included an upgrade to scala-parallel-collections version 1.0.3, the changes compared to 0.2.0 are minor. - removed OSGi metadata - renamed some internal inner classes - added `Automatic-Module-Name` ### Why are the changes needed? According to the posts, this solves issues for developers that write unit tests for their applications. Stephen Coy suggested to use the https://www.mojohaus.org/flatten-maven-plugin. While this sounds like a more principled solution, it is possibly too risky to do at this specific point in time? ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Locally Closes #33948 from lrytz/parCollDep. Authored-by: Lukas Rytz <lukas.rytz@gmail.com> Signed-off-by: Sean Owen <srowen@gmail.com>
This commit is contained in:
parent
bd62ad9982
commit
1a62e6a2c1
15
core/pom.xml
15
core/pom.xml
|
@ -35,6 +35,12 @@
|
|||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
|
@ -639,15 +645,6 @@
|
|||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
</project>
|
||||
|
|
|
@ -54,11 +54,15 @@ sed_i() {
|
|||
sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
|
||||
}
|
||||
|
||||
export -f sed_i
|
||||
|
||||
BASEDIR=$(dirname $0)/..
|
||||
find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \
|
||||
-exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \;
|
||||
for f in $(find "$BASEDIR" -name 'pom.xml' -not -path '*target*'); do
|
||||
echo $f
|
||||
sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' $f
|
||||
sed_i 's/^\([[:space:]]*<!-- #if scala-'$TO_VERSION' -->\)<!--/\1/' $f
|
||||
sed_i 's/^\([[:space:]]*\)-->\(<!-- #endif scala-'$TO_VERSION' -->\)/\1\2/' $f
|
||||
sed_i 's/^\([[:space:]]*<!-- #if scala-'$FROM_VERSION' -->\)/\1<!--/' $f
|
||||
sed_i 's/^\([[:space:]]*\)\(<!-- #endif scala-'$FROM_VERSION' -->\)/\1-->\2/' $f
|
||||
done
|
||||
|
||||
# dependency:get is workaround for SPARK-34762 to download the JAR file of commons-cli.
|
||||
# Without this, build with Scala 2.13 using SBT will fail because the help plugin used below downloads only the POM file.
|
||||
|
|
17
external/avro/pom.xml
vendored
17
external/avro/pom.xml
vendored
|
@ -70,22 +70,17 @@
|
|||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.tukaani</groupId>
|
||||
<artifactId>xz</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
<build>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
|
|
17
external/kafka-0-10-sql/pom.xml
vendored
17
external/kafka-0-10-sql/pom.xml
vendored
|
@ -74,6 +74,12 @@
|
|||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
|
@ -169,17 +175,6 @@
|
|||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
<build>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
|
|
18
external/kafka-0-10/pom.xml
vendored
18
external/kafka-0-10/pom.xml
vendored
|
@ -59,6 +59,12 @@
|
|||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
|
@ -136,18 +142,6 @@
|
|||
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
|
|
|
@ -91,6 +91,12 @@
|
|||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.scalanlp</groupId>
|
||||
<artifactId>breeze_${scala.binary.version}</artifactId>
|
||||
|
@ -157,18 +163,6 @@
|
|||
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
|
|
16
pom.xml
16
pom.xml
|
@ -399,6 +399,13 @@
|
|||
<version>${project.version}</version>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
<version>1.0.3</version>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>com.twitter</groupId>
|
||||
<artifactId>chill_${scala.binary.version}</artifactId>
|
||||
|
@ -3370,15 +3377,6 @@
|
|||
<scala.version>2.13.5</scala.version>
|
||||
<scala.binary.version>2.13</scala.binary.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
<version>0.2.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
|
|
|
@ -87,6 +87,12 @@
|
|||
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.scalacheck</groupId>
|
||||
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
||||
|
@ -191,16 +197,4 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
|
@ -90,6 +90,12 @@
|
|||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.orc</groupId>
|
||||
<artifactId>orc-core</artifactId>
|
||||
|
@ -252,15 +258,6 @@
|
|||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>hadoop-2.7</id>
|
||||
<dependencies>
|
||||
|
|
|
@ -61,6 +61,12 @@
|
|||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
|
@ -138,17 +144,6 @@
|
|||
<artifactId>commons-cli</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
<build>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
|
|
|
@ -79,6 +79,12 @@
|
|||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
|
@ -217,15 +223,6 @@
|
|||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
|
|
|
@ -50,6 +50,12 @@
|
|||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
<!-- #if scala-2.13 --><!--
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
--><!-- #endif scala-2.13 -->
|
||||
|
||||
<!--
|
||||
This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude
|
||||
|
@ -132,15 +138,4 @@
|
|||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>scala-2.13</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang.modules</groupId>
|
||||
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
Loading…
Reference in a new issue