daaca14c16
Let's give this another go using a version of Hive that shades its JLine dependency. Author: Prashant Sharma <prashant.s@imaginea.com> Author: Patrick Wendell <pwendell@gmail.com> Closes #3159 from pwendell/scala-2.11-prashant and squashes the following commits: e93aa3e [Patrick Wendell] Restoring -Phive-thriftserver profile and cleaning up build script. f65d17d [Patrick Wendell] Fixing build issue due to merge conflict a8c41eb [Patrick Wendell] Reverting dev/run-tests back to master state. 7a6eb18 [Patrick Wendell] Merge remote-tracking branch 'apache/master' into scala-2.11-prashant 583aa07 [Prashant Sharma] REVERT ME: removed hive thirftserver 3680e58 [Prashant Sharma] Revert "REVERT ME: Temporarily removing some Cli tests." 935fb47 [Prashant Sharma] Revert "Fixed by disabling a few tests temporarily." 925e90f [Prashant Sharma] Fixed by disabling a few tests temporarily. 2fffed3 [Prashant Sharma] Exclude groovy from sbt build, and also provide a way for such instances in future. 8bd4e40 [Prashant Sharma] Switched to gmaven plus, it fixes random failures observer with its predecessor gmaven. 5272ce5 [Prashant Sharma] SPARK_SCALA_VERSION related bugs. 2121071 [Patrick Wendell] Migrating version detection to PySpark b1ed44d [Patrick Wendell] REVERT ME: Temporarily removing some Cli tests. 1743a73 [Patrick Wendell] Removing decimal test that doesn't work with Scala 2.11 f5cad4e [Patrick Wendell] Add Scala 2.11 docs 210d7e1 [Patrick Wendell] Revert "Testing new Hive version with shaded jline" 48518ce [Patrick Wendell] Remove association of Hive and Thriftserver profiles. e9d0a06 [Patrick Wendell] Revert "Enable thritfserver for Scala 2.10 only" 67ec364 [Patrick Wendell] Guard building of thriftserver around Scala 2.10 check 8502c23 [Patrick Wendell] Enable thritfserver for Scala 2.10 only e22b104 [Patrick Wendell] Small fix in pom file ec402ab [Patrick Wendell] Various fixes 0be5a9d [Patrick Wendell] Testing new Hive version with shaded jline 4eaec65 [Prashant Sharma] Changed scripts to ignore target. 5167bea [Prashant Sharma] small correction a4fcac6 [Prashant Sharma] Run against scala 2.11 on jenkins. 80285f4 [Prashant Sharma] MAven equivalent of setting spark.executor.extraClasspath during tests. 034b369 [Prashant Sharma] Setting test jars on executor classpath during tests from sbt. d4874cb [Prashant Sharma] Fixed Python Runner suite. null check should be first case in scala 2.11. 6f50f13 [Prashant Sharma] Fixed build after rebasing with master. We should use ${scala.binary.version} instead of just 2.10 e56ca9d [Prashant Sharma] Print an error if build for 2.10 and 2.11 is spotted. 937c0b8 [Prashant Sharma] SCALA_VERSION -> SPARK_SCALA_VERSION cb059b0 [Prashant Sharma] Code review 0476e5e [Prashant Sharma] Scala 2.11 support with repl and all build changes.
369 lines
14 KiB
XML
369 lines
14 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent</artifactId>
|
|
<version>1.2.0-SNAPSHOT</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-assembly_2.10</artifactId>
|
|
<name>Spark Project Assembly</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<packaging>pom</packaging>
|
|
|
|
<properties>
|
|
<sbt.project.name>assembly</sbt.project.name>
|
|
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
|
|
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
|
|
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>
|
|
<deb.pkg.name>spark</deb.pkg.name>
|
|
<deb.install.path>/usr/share/spark</deb.install.path>
|
|
<deb.user>root</deb.user>
|
|
<deb.bin.filemode>744</deb.bin.filemode>
|
|
</properties>
|
|
|
|
<dependencies>
|
|
<!-- Promote Guava to compile scope in this module so it's included while shading. -->
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-bagel_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-mllib_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-repl_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-deploy-plugin</artifactId>
|
|
<configuration>
|
|
<skip>true</skip>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-install-plugin</artifactId>
|
|
<configuration>
|
|
<skip>true</skip>
|
|
</configuration>
|
|
</plugin>
|
|
<!-- Use the shade plugin to create a big JAR with all the dependencies -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<configuration>
|
|
<shadedArtifactAttached>false</shadedArtifactAttached>
|
|
<outputFile>${spark.jar}</outputFile>
|
|
<artifactSet>
|
|
<includes>
|
|
<include>*:*</include>
|
|
</includes>
|
|
</artifactSet>
|
|
<filters>
|
|
<filter>
|
|
<artifact>*:*</artifact>
|
|
<excludes>
|
|
<exclude>org/datanucleus/**</exclude>
|
|
<exclude>META-INF/*.SF</exclude>
|
|
<exclude>META-INF/*.DSA</exclude>
|
|
<exclude>META-INF/*.RSA</exclude>
|
|
</excludes>
|
|
</filter>
|
|
</filters>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
<configuration>
|
|
<relocations>
|
|
<relocation>
|
|
<pattern>com.google</pattern>
|
|
<shadedPattern>org.spark-project.guava</shadedPattern>
|
|
<includes>
|
|
<include>com.google.common.**</include>
|
|
</includes>
|
|
<excludes>
|
|
<exclude>com/google/common/base/Absent*</exclude>
|
|
<exclude>com/google/common/base/Optional*</exclude>
|
|
<exclude>com/google/common/base/Present*</exclude>
|
|
</excludes>
|
|
</relocation>
|
|
</relocations>
|
|
<transformers>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
|
|
<resource>META-INF/services/org.apache.hadoop.fs.FileSystem</resource>
|
|
</transformer>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
|
|
<resource>reference.conf</resource>
|
|
</transformer>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
|
|
<resource>log4j.properties</resource>
|
|
</transformer>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
|
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
|
|
</transformers>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<profiles>
|
|
<profile>
|
|
<id>yarn-alpha</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-yarn-alpha_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>yarn</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>hive</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>hive-thriftserver</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>spark-ganglia-lgpl</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-ganglia-lgpl_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
<profile>
|
|
<id>bigtop-dist</id>
|
|
<!-- This profile uses the assembly plugin to create a special "dist" package for BigTop
|
|
that contains Spark but not the Hadoop JARs it depends on. -->
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-assembly-plugin</artifactId>
|
|
<version>2.4</version>
|
|
<executions>
|
|
<execution>
|
|
<id>dist</id>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>single</goal>
|
|
</goals>
|
|
<configuration>
|
|
<descriptors>
|
|
<descriptor>src/main/assembly/assembly.xml</descriptor>
|
|
</descriptors>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</profile>
|
|
<profile>
|
|
<id>deb</id>
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>buildnumber-maven-plugin</artifactId>
|
|
<version>1.2</version>
|
|
<executions>
|
|
<execution>
|
|
<phase>validate</phase>
|
|
<goals>
|
|
<goal>create</goal>
|
|
</goals>
|
|
<configuration>
|
|
<shortRevisionLength>8</shortRevisionLength>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.vafer</groupId>
|
|
<artifactId>jdeb</artifactId>
|
|
<version>0.11</version>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>jdeb</goal>
|
|
</goals>
|
|
<configuration>
|
|
<deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
|
|
<attach>false</attach>
|
|
<compression>gzip</compression>
|
|
<dataSet>
|
|
<data>
|
|
<src>${spark.jar}</src>
|
|
<type>file</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}/jars</prefix>
|
|
</mapper>
|
|
</data>
|
|
<data>
|
|
<src>${basedir}/src/deb/RELEASE</src>
|
|
<type>file</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}</prefix>
|
|
</mapper>
|
|
</data>
|
|
<data>
|
|
<src>${basedir}/../conf</src>
|
|
<type>directory</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}/conf</prefix>
|
|
<filemode>744</filemode>
|
|
</mapper>
|
|
</data>
|
|
<data>
|
|
<src>${basedir}/../bin</src>
|
|
<type>directory</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}/bin</prefix>
|
|
<filemode>${deb.bin.filemode}</filemode>
|
|
</mapper>
|
|
</data>
|
|
<data>
|
|
<src>${basedir}/../sbin</src>
|
|
<type>directory</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}/sbin</prefix>
|
|
<filemode>744</filemode>
|
|
</mapper>
|
|
</data>
|
|
<data>
|
|
<src>${basedir}/../python</src>
|
|
<type>directory</type>
|
|
<mapper>
|
|
<type>perm</type>
|
|
<user>${deb.user}</user>
|
|
<group>${deb.user}</group>
|
|
<prefix>${deb.install.path}/python</prefix>
|
|
<filemode>744</filemode>
|
|
</mapper>
|
|
</data>
|
|
</dataSet>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</profile>
|
|
<profile>
|
|
<id>kinesis-asl</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
<version>${commons.httpclient.version}</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
</profiles>
|
|
</project>
|