[SPARK-4121] Set commons-math3 version based on hadoop profiles, instead of shading
In #2928 , we shade commons-math3 to prevent future conflicts with hadoop. It caused problems with our Jenkins master build with maven. Some tests used local-cluster mode, where the assembly jar contains relocated math3 classes, while mllib test code still compiles with core and the untouched math3 classes. This PR sets commons-math3 version based on hadoop profiles. pwendell JoshRosen srowen Author: Xiangrui Meng <meng@databricks.com> Closes #3023 from mengxr/SPARK-4121-alt and squashes the following commits: 580f6d9 [Xiangrui Meng] replace tab by spaces 7f71f08 [Xiangrui Meng] revert changes to PoissonSampler to avoid conflicts d3353d9 [Xiangrui Meng] do not shade commons-math3 b4180dc [Xiangrui Meng] temp work
This commit is contained in:
parent
7894de276b
commit
d8176b1c2f
|
@ -146,10 +146,6 @@
|
|||
<exclude>com/google/common/base/Present*</exclude>
|
||||
</excludes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.apache.commons.math3</pattern>
|
||||
<shadedPattern>org.spark-project.commons.math3</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
|
||||
|
|
|
@ -65,6 +65,10 @@
|
|||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
9
pom.xml
9
pom.xml
|
@ -145,6 +145,7 @@
|
|||
<aws.java.sdk.version>1.8.3</aws.java.sdk.version>
|
||||
<aws.kinesis.client.version>1.1.0</aws.kinesis.client.version>
|
||||
<commons.httpclient.version>4.2.6</commons.httpclient.version>
|
||||
<commons.math3.version>3.1.1</commons.math3.version>
|
||||
|
||||
<PermGen>64m</PermGen>
|
||||
<MaxPermGen>512m</MaxPermGen>
|
||||
|
@ -318,7 +319,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<version>3.3</version>
|
||||
<version>${commons.math3.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
|
@ -1175,6 +1176,10 @@
|
|||
|
||||
<!-- A series of build profiles where customizations for particular Hadoop releases can be made -->
|
||||
|
||||
<!-- Hadoop-a.b.c dependencies can be found at
|
||||
http://hadoop.apache.org/docs/ra.b.c/hadoop-project-dist/hadoop-common/dependency-analysis.html
|
||||
-->
|
||||
|
||||
<profile>
|
||||
<id>hadoop-0.23</id>
|
||||
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a Hadoop 0.23.X issue -->
|
||||
|
@ -1204,6 +1209,7 @@
|
|||
<hadoop.version>2.3.0</hadoop.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<jets3t.version>0.9.0</jets3t.version>
|
||||
<commons.math3.version>3.1.1</commons.math3.version>
|
||||
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
|
||||
</properties>
|
||||
</profile>
|
||||
|
@ -1214,6 +1220,7 @@
|
|||
<hadoop.version>2.4.0</hadoop.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<jets3t.version>0.9.0</jets3t.version>
|
||||
<commons.math3.version>3.1.1</commons.math3.version>
|
||||
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
Loading…
Reference in a new issue