ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-27721][BUILD] Switch to use right leveldbjni according to the platforms

Browse source 
This change adds a profile to switch to use the right leveldbjni package according to the platforms:
aarch64 uses org.openlabtesting.leveldbjni:leveldbjni-all.1.8, and other platforms use the old one org.fusesource.leveldbjni:leveldbjni-all.1.8.
And because some hadoop dependencies packages are also depend on org.fusesource.leveldbjni:leveldbjni-all, but hadoop merge the similar change on trunk, details see
https://issues.apache.org/jira/browse/HADOOP-16614, so exclude the dependency of org.fusesource.leveldbjni for these hadoop packages related.
Then Spark can build/test on aarch64 platform successfully.

Closes #26636 from huangtianhua/add-aarch64-leveldbjni.

Authored-by: huangtianhua <huangtianhua@huawei.com>
Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
huangtianhua 2019-12-02 09:04:00 -06:00 committed by Sean Owen
parent 54edaee586
commit e842033acc
3 changed files with 24 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common/kvstore/pom.xml
View file

@ -45,7 +45,7 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.fusesource.leveldbjni</groupId>
<groupId>${leveldbjni.group}</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
<dependency>

2
common/network-common/pom.xml
View file

@ -52,7 +52,7 @@
</dependency>
<dependency>
<groupId>org.fusesource.leveldbjni</groupId>
<groupId>${leveldbjni.group}</groupId>
<artifactId>leveldbjni-all</artifactId>
<version>1.8</version>
</dependency>

23
pom.xml
View file

@ -203,6 +203,8 @@
and ./python/setup.py too.
-->
<arrow.version>0.15.1</arrow.version>
<!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
<leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
<test.java.home>${java.home}</test.java.home>
<test.exclude.tags></test.exclude.tags>
@ -531,7 +533,7 @@
<version>${commons.httpcore.version}</version>
</dependency>
<dependency>
<groupId>org.fusesource.leveldbjni</groupId>
<groupId>${leveldbjni.group}</groupId>
<artifactId>leveldbjni-all</artifactId>
<version>1.8</version>
</dependency>
@ -958,6 +960,10 @@
<version>${hadoop.version}</version>
<scope>${hadoop.deps.scope}</scope>
<exclusions>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
@ -1204,6 +1210,10 @@
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
@ -1244,6 +1254,10 @@
<version>${yarn.version}</version>
<scope>${hadoop.deps.scope}</scope>
<exclusions>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
@ -3118,5 +3132,12 @@
<profile>
<id>sparkr</id>
</profile>
<!-- use org.openlabtesting.leveldbjni on aarch64 platform -->
<profile>
<id>aarch64</id>
<properties>
<leveldbjni.group>org.openlabtesting.leveldbjni</leveldbjni.group>
</properties>
</profile>
</profiles>
</project>