2012-11-20 18:50:07 -05:00
|
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
2013-07-16 20:21:33 -04:00
|
|
|
<!--
|
|
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
|
|
~ this work for additional information regarding copyright ownership.
|
|
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
|
|
~ the License. You may obtain a copy of the License at
|
|
|
|
~
|
|
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
~
|
|
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
~ See the License for the specific language governing permissions and
|
|
|
|
~ limitations under the License.
|
|
|
|
-->
|
|
|
|
|
2012-11-20 18:50:07 -05:00
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
|
|
<modelVersion>4.0.0</modelVersion>
|
2013-09-02 21:34:03 -04:00
|
|
|
<parent>
|
|
|
|
<groupId>org.apache</groupId>
|
|
|
|
<artifactId>apache</artifactId>
|
2013-10-15 02:10:54 -04:00
|
|
|
<version>13</version>
|
2013-09-02 21:34:03 -04:00
|
|
|
</parent>
|
2013-08-31 22:27:07 -04:00
|
|
|
<groupId>org.apache.spark</groupId>
|
2013-03-15 14:41:51 -04:00
|
|
|
<artifactId>spark-parent</artifactId>
|
2013-09-24 14:41:51 -04:00
|
|
|
<version>0.9.0-incubating-SNAPSHOT</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
<packaging>pom</packaging>
|
|
|
|
<name>Spark Project Parent POM</name>
|
2013-08-31 22:27:07 -04:00
|
|
|
<url>http://spark.incubator.apache.org/</url>
|
2012-11-20 18:50:07 -05:00
|
|
|
<licenses>
|
|
|
|
<license>
|
2013-08-31 22:27:07 -04:00
|
|
|
<name>Apache 2.0 License</name>
|
|
|
|
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
|
2012-11-20 18:50:07 -05:00
|
|
|
<distribution>repo</distribution>
|
|
|
|
</license>
|
|
|
|
</licenses>
|
|
|
|
<scm>
|
2013-08-31 22:27:07 -04:00
|
|
|
<connection>scm:git:git@github.com:apache/incubator-spark.git</connection>
|
2013-09-12 16:47:09 -04:00
|
|
|
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-spark.git</developerConnection>
|
2013-08-31 22:27:07 -04:00
|
|
|
<url>scm:git:git@github.com:apache/incubator-spark.git</url>
|
2013-10-05 20:53:11 -04:00
|
|
|
<tag>HEAD</tag>
|
2012-11-20 18:50:07 -05:00
|
|
|
</scm>
|
|
|
|
<developers>
|
|
|
|
<developer>
|
|
|
|
<id>matei</id>
|
|
|
|
<name>Matei Zaharia</name>
|
|
|
|
<email>matei.zaharia@gmail.com</email>
|
|
|
|
<url>http://www.cs.berkeley.edu/~matei</url>
|
2013-09-01 01:45:53 -04:00
|
|
|
<organization>Apache Software Foundation</organization>
|
|
|
|
<organizationUrl>http://spark.incubator.apache.org</organizationUrl>
|
2012-11-20 18:50:07 -05:00
|
|
|
</developer>
|
|
|
|
</developers>
|
|
|
|
<issueManagement>
|
2013-08-31 22:27:07 -04:00
|
|
|
<system>JIRA</system>
|
2012-12-10 13:58:56 -05:00
|
|
|
<url>https://spark-project.atlassian.net/browse/SPARK</url>
|
2012-11-20 18:50:07 -05:00
|
|
|
</issueManagement>
|
|
|
|
|
|
|
|
<prerequisites>
|
|
|
|
<maven>3.0.0</maven>
|
|
|
|
</prerequisites>
|
|
|
|
|
2013-10-15 02:10:54 -04:00
|
|
|
<mailingLists>
|
|
|
|
<mailingList>
|
|
|
|
<name>Dev Mailing List</name>
|
|
|
|
<post>dev@spark.incubator.apache.org</post>
|
|
|
|
<subscribe>dev-subscribe@spark.incubator.apache.org</subscribe>
|
|
|
|
<unsubscribe>dev-unsubscribe@spark.incubator.apache.org</unsubscribe>
|
|
|
|
</mailingList>
|
|
|
|
|
|
|
|
<mailingList>
|
|
|
|
<name>User Mailing List</name>
|
|
|
|
<post>user@spark.incubator.apache.org</post>
|
|
|
|
<subscribe>user-subscribe@spark.incubator.apache.org</subscribe>
|
|
|
|
<unsubscribe>user-unsubscribe@spark.incubator.apache.org</unsubscribe>
|
|
|
|
</mailingList>
|
|
|
|
|
|
|
|
<mailingList>
|
|
|
|
<name>Commits Mailing List</name>
|
|
|
|
<post>commits@spark.incubator.apache.org</post>
|
|
|
|
<subscribe>commits-subscribe@spark.incubator.apache.org</subscribe>
|
|
|
|
<unsubscribe>commits-unsubscribe@spark.incubator.apache.org</unsubscribe>
|
|
|
|
</mailingList>
|
|
|
|
</mailingLists>
|
|
|
|
|
2012-11-20 18:50:07 -05:00
|
|
|
<modules>
|
|
|
|
<module>core</module>
|
|
|
|
<module>bagel</module>
|
2012-12-18 13:44:03 -05:00
|
|
|
<module>examples</module>
|
2013-07-30 17:03:15 -04:00
|
|
|
<module>mllib</module>
|
2013-07-18 01:59:00 -04:00
|
|
|
<module>tools</module>
|
2013-01-20 22:22:24 -05:00
|
|
|
<module>streaming</module>
|
2012-12-18 13:44:03 -05:00
|
|
|
<module>repl</module>
|
2013-08-27 22:23:54 -04:00
|
|
|
<module>assembly</module>
|
2012-11-20 18:50:07 -05:00
|
|
|
</modules>
|
|
|
|
|
|
|
|
<properties>
|
|
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
|
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
|
|
|
|
2012-11-27 20:50:42 -05:00
|
|
|
<java.version>1.5</java.version>
|
2013-04-18 17:58:38 -04:00
|
|
|
<scala.version>2.9.3</scala.version>
|
2013-09-15 15:46:26 -04:00
|
|
|
<mesos.version>0.13.0</mesos.version>
|
2013-08-21 16:50:24 -04:00
|
|
|
<akka.version>2.0.5</akka.version>
|
2013-06-30 20:11:14 -04:00
|
|
|
<slf4j.version>1.7.2</slf4j.version>
|
2013-03-15 14:41:51 -04:00
|
|
|
<log4j.version>1.2.17</log4j.version>
|
2013-08-21 16:50:24 -04:00
|
|
|
<hadoop.version>1.0.4</hadoop.version>
|
2013-09-13 17:06:45 -04:00
|
|
|
<yarn.version>0.23.7</yarn.version>
|
2013-08-27 22:23:54 -04:00
|
|
|
<hbase.version>0.94.6</hbase.version>
|
2013-05-29 23:14:59 -04:00
|
|
|
|
2013-06-12 20:20:05 -04:00
|
|
|
<PermGen>64m</PermGen>
|
2013-05-29 23:14:59 -04:00
|
|
|
<MaxPermGen>512m</MaxPermGen>
|
2012-11-20 18:50:07 -05:00
|
|
|
</properties>
|
|
|
|
|
|
|
|
<repositories>
|
|
|
|
<repository>
|
|
|
|
<id>jboss-repo</id>
|
|
|
|
<name>JBoss Repository</name>
|
|
|
|
<url>http://repository.jboss.org/nexus/content/repositories/releases/</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</repository>
|
|
|
|
<repository>
|
|
|
|
<id>cloudera-repo</id>
|
|
|
|
<name>Cloudera Repository</name>
|
|
|
|
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</repository>
|
|
|
|
<repository>
|
2013-02-24 19:24:52 -05:00
|
|
|
<id>akka-repo</id>
|
|
|
|
<name>Akka Repository</name>
|
|
|
|
<url>http://repo.akka.io/releases/</url>
|
2012-11-20 18:50:07 -05:00
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</repository>
|
|
|
|
</repositories>
|
|
|
|
<pluginRepositories>
|
|
|
|
<pluginRepository>
|
|
|
|
<id>oss-sonatype-releases</id>
|
|
|
|
<name>OSS Sonatype</name>
|
|
|
|
<url>https://oss.sonatype.org/content/repositories/releases</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</pluginRepository>
|
|
|
|
<pluginRepository>
|
|
|
|
<id>oss-sonatype-snapshots</id>
|
|
|
|
<name>OSS Sonatype</name>
|
|
|
|
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</pluginRepository>
|
|
|
|
<pluginRepository>
|
|
|
|
<id>oss-sonatype</id>
|
|
|
|
<name>OSS Sonatype</name>
|
|
|
|
<url>https://oss.sonatype.org/content/groups/public</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</pluginRepository>
|
|
|
|
</pluginRepositories>
|
|
|
|
|
|
|
|
<dependencyManagement>
|
|
|
|
<dependencies>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-server</artifactId>
|
2013-08-21 16:50:24 -04:00
|
|
|
<version>7.6.8.v20121106</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.google.guava</groupId>
|
|
|
|
<artifactId>guava</artifactId>
|
2013-08-21 16:50:24 -04:00
|
|
|
<version>14.0.1</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
|
|
<artifactId>jsr305</artifactId>
|
|
|
|
<version>1.3.9</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.slf4j</groupId>
|
|
|
|
<artifactId>slf4j-api</artifactId>
|
|
|
|
<version>${slf4j.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.slf4j</groupId>
|
|
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
|
|
<version>${slf4j.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.slf4j</groupId>
|
|
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
|
|
<version>${slf4j.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.ning</groupId>
|
|
|
|
<artifactId>compress-lzf</artifactId>
|
|
|
|
<version>0.8.4</version>
|
|
|
|
</dependency>
|
2013-07-30 20:25:42 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.xerial.snappy</groupId>
|
|
|
|
<artifactId>snappy-java</artifactId>
|
|
|
|
<version>1.0.5</version>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
<dependency>
|
2013-06-19 07:34:30 -04:00
|
|
|
<groupId>org.ow2.asm</groupId>
|
|
|
|
<artifactId>asm</artifactId>
|
|
|
|
<version>4.0</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependency>
|
2013-10-18 17:10:24 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.clearspring.analytics</groupId>
|
|
|
|
<artifactId>stream</artifactId>
|
|
|
|
<version>2.4.0</version>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.google.protobuf</groupId>
|
|
|
|
<artifactId>protobuf-java</artifactId>
|
|
|
|
<version>2.4.1</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
2013-07-24 23:55:48 -04:00
|
|
|
<groupId>com.twitter</groupId>
|
2013-07-25 11:58:02 -04:00
|
|
|
<artifactId>chill_2.9.3</artifactId>
|
2013-08-08 16:30:27 -04:00
|
|
|
<version>0.3.1</version>
|
2013-07-24 23:55:48 -04:00
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.twitter</groupId>
|
|
|
|
<artifactId>chill-java</artifactId>
|
2013-08-08 16:30:27 -04:00
|
|
|
<version>0.3.1</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.typesafe.akka</groupId>
|
|
|
|
<artifactId>akka-actor</artifactId>
|
|
|
|
<version>${akka.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.typesafe.akka</groupId>
|
|
|
|
<artifactId>akka-remote</artifactId>
|
|
|
|
<version>${akka.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.typesafe.akka</groupId>
|
|
|
|
<artifactId>akka-slf4j</artifactId>
|
|
|
|
<version>${akka.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>it.unimi.dsi</groupId>
|
|
|
|
<artifactId>fastutil</artifactId>
|
|
|
|
<version>6.4.4</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>colt</groupId>
|
|
|
|
<artifactId>colt</artifactId>
|
|
|
|
<version>1.2.0</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.github.scala-incubator.io</groupId>
|
2013-04-18 17:58:38 -04:00
|
|
|
<artifactId>scala-io-file_2.9.2</artifactId>
|
2012-11-20 18:50:07 -05:00
|
|
|
<version>0.4.1</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.mesos</groupId>
|
|
|
|
<artifactId>mesos</artifactId>
|
2012-11-27 18:43:30 -05:00
|
|
|
<version>${mesos.version}</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependency>
|
2013-05-16 17:31:26 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>io.netty</groupId>
|
|
|
|
<artifactId>netty-all</artifactId>
|
|
|
|
<version>4.0.0.Beta2</version>
|
|
|
|
</dependency>
|
2013-05-15 02:28:34 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.derby</groupId>
|
|
|
|
<artifactId>derby</artifactId>
|
|
|
|
<version>10.4.2.0</version>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2013-08-26 14:05:14 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>net.liftweb</groupId>
|
|
|
|
<artifactId>lift-json_2.9.2</artifactId>
|
|
|
|
<version>2.5</version>
|
|
|
|
</dependency>
|
2013-07-24 19:12:32 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.codahale.metrics</groupId>
|
|
|
|
<artifactId>metrics-core</artifactId>
|
|
|
|
<version>3.0.0</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.codahale.metrics</groupId>
|
|
|
|
<artifactId>metrics-jvm</artifactId>
|
|
|
|
<version>3.0.0</version>
|
|
|
|
</dependency>
|
2013-08-06 04:19:37 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.codahale.metrics</groupId>
|
|
|
|
<artifactId>metrics-json</artifactId>
|
|
|
|
<version>3.0.0</version>
|
|
|
|
</dependency>
|
2013-09-09 12:32:18 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.codahale.metrics</groupId>
|
|
|
|
<artifactId>metrics-ganglia</artifactId>
|
|
|
|
<version>3.0.0</version>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scala-lang</groupId>
|
|
|
|
<artifactId>scala-compiler</artifactId>
|
|
|
|
<version>${scala.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.scala-lang</groupId>
|
|
|
|
<artifactId>jline</artifactId>
|
|
|
|
<version>${scala.version}</version>
|
|
|
|
</dependency>
|
2013-07-08 19:07:09 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scala-lang</groupId>
|
|
|
|
<artifactId>scala-library</artifactId>
|
|
|
|
<version>${scala.version}</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.scala-lang</groupId>
|
|
|
|
<artifactId>scalap</artifactId>
|
|
|
|
<version>${scala.version}</version>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
|
2013-03-15 14:41:51 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>log4j</groupId>
|
|
|
|
<artifactId>log4j</artifactId>
|
|
|
|
<version>${log4j.version}</version>
|
|
|
|
</dependency>
|
|
|
|
|
2012-11-20 18:50:07 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scalatest</groupId>
|
2013-10-05 20:53:11 -04:00
|
|
|
<artifactId>scalatest_2.9.3</artifactId>
|
2013-04-18 17:58:38 -04:00
|
|
|
<version>1.9.1</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2013-01-29 01:40:16 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.easymock</groupId>
|
|
|
|
<artifactId>easymock</artifactId>
|
|
|
|
<version>3.1</version>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scalacheck</groupId>
|
2013-10-05 20:53:11 -04:00
|
|
|
<artifactId>scalacheck_2.9.3</artifactId>
|
2013-04-18 17:58:38 -04:00
|
|
|
<version>1.10.0</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.novocode</groupId>
|
|
|
|
<artifactId>junit-interface</artifactId>
|
2013-08-21 16:50:24 -04:00
|
|
|
<version>0.9</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
Standalone Scheduler fault tolerance using ZooKeeper
This patch implements full distributed fault tolerance for standalone scheduler Masters.
There is only one master Leader at a time, which is actively serving scheduling
requests. If this Leader crashes, another master will eventually be elected, reconstruct
the state from the first Master, and continue serving scheduling requests.
Leader election is performed using the ZooKeeper leader election pattern. We try to minimize
the use of ZooKeeper and the assumptions about ZooKeeper's behavior, so there is a layer of
retries and session monitoring on top of the ZooKeeper client.
Master failover follows directly from the single-node Master recovery via the file
system (patch 194ba4b8), save that the Master state is stored in ZooKeeper instead.
Configuration:
By default, no recovery mechanism is enabled (spark.deploy.recoveryMode = NONE).
By setting spark.deploy.recoveryMode to ZOOKEEPER and setting spark.deploy.zookeeper.url
to an appropriate ZooKeeper URL, ZooKeeper recovery mode is enabled.
By setting spark.deploy.recoveryMode to FILESYSTEM and setting spark.deploy.recoveryDirectory
to an appropriate directory accessible by the Master, we will keep the behavior of from 194ba4b8.
Additionally, places where a Master could be specificied by a spark:// url can now take
comma-delimited lists to specify backup masters. Note that this is only used for registration
of NEW Workers and application Clients. Once a Worker or Client has registered with the
Master Leader, it is "in the system" and will never need to register again.
Forthcoming:
Documentation, tests (! - only ad hoc testing has been performed so far)
I do not intend for this commit to be merged until tests are added, but this patch should
still be mostly reviewable until then.
2013-09-19 17:40:14 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
|
|
<artifactId>zookeeper</artifactId>
|
|
|
|
<version>3.4.5</version>
|
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
2013-08-15 15:10:31 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.hadoop</groupId>
|
|
|
|
<artifactId>hadoop-client</artifactId>
|
2013-08-15 18:31:31 -04:00
|
|
|
<version>${hadoop.version}</version>
|
2013-08-15 15:10:31 -04:00
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>asm</groupId>
|
|
|
|
<artifactId>asm</artifactId>
|
|
|
|
</exclusion>
|
2013-08-15 19:49:24 -04:00
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-jaxrs</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-xc</artifactId>
|
|
|
|
</exclusion>
|
2013-08-15 15:10:31 -04:00
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
2013-09-10 02:39:25 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>net.java.dev.jets3t</groupId>
|
|
|
|
<artifactId>jets3t</artifactId>
|
|
|
|
<version>0.7.1</version>
|
|
|
|
</dependency>
|
2013-09-06 13:35:06 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.hadoop</groupId>
|
|
|
|
<artifactId>hadoop-yarn-api</artifactId>
|
2013-09-13 17:06:45 -04:00
|
|
|
<version>${yarn.version}</version>
|
2013-09-06 13:35:06 -04:00
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>asm</groupId>
|
|
|
|
<artifactId>asm</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-jaxrs</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-xc</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.hadoop</groupId>
|
|
|
|
<artifactId>hadoop-yarn-common</artifactId>
|
2013-09-13 17:06:45 -04:00
|
|
|
<version>${yarn.version}</version>
|
2013-09-06 13:35:06 -04:00
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>asm</groupId>
|
|
|
|
<artifactId>asm</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-jaxrs</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-xc</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.hadoop</groupId>
|
|
|
|
<artifactId>hadoop-yarn-client</artifactId>
|
2013-09-13 17:06:45 -04:00
|
|
|
<version>${yarn.version}</version>
|
2013-09-06 13:35:06 -04:00
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>asm</groupId>
|
|
|
|
<artifactId>asm</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-jaxrs</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
|
|
<artifactId>jackson-xc</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
2013-08-15 18:31:31 -04:00
|
|
|
<!-- Specify Avro version because Kafka also has it as a dependency -->
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.avro</groupId>
|
|
|
|
<artifactId>avro</artifactId>
|
|
|
|
<version>1.7.4</version>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.avro</groupId>
|
|
|
|
<artifactId>avro-ipc</artifactId>
|
|
|
|
<version>1.7.4</version>
|
|
|
|
<exclusions>
|
|
|
|
<exclusion>
|
|
|
|
<groupId>org.jboss.netty</groupId>
|
|
|
|
<artifactId>netty</artifactId>
|
|
|
|
</exclusion>
|
|
|
|
</exclusions>
|
|
|
|
</dependency>
|
2012-11-20 18:50:07 -05:00
|
|
|
</dependencies>
|
|
|
|
</dependencyManagement>
|
|
|
|
|
|
|
|
<build>
|
|
|
|
<pluginManagement>
|
|
|
|
<plugins>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-enforcer-plugin</artifactId>
|
|
|
|
<version>1.1.1</version>
|
|
|
|
<executions>
|
|
|
|
<execution>
|
|
|
|
<id>enforce-versions</id>
|
|
|
|
<goals>
|
|
|
|
<goal>enforce</goal>
|
|
|
|
</goals>
|
|
|
|
<configuration>
|
|
|
|
<rules>
|
|
|
|
<requireMavenVersion>
|
|
|
|
<version>3.0.0</version>
|
|
|
|
</requireMavenVersion>
|
|
|
|
<requireJavaVersion>
|
2012-11-27 18:43:30 -05:00
|
|
|
<version>${java.version}</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
</requireJavaVersion>
|
|
|
|
</rules>
|
|
|
|
</configuration>
|
|
|
|
</execution>
|
|
|
|
</executions>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
|
|
<version>1.7</version>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>net.alchim31.maven</groupId>
|
|
|
|
<artifactId>scala-maven-plugin</artifactId>
|
2013-07-16 14:52:20 -04:00
|
|
|
<version>3.1.5</version>
|
2012-11-20 18:50:07 -05:00
|
|
|
<executions>
|
|
|
|
<execution>
|
|
|
|
<id>scala-compile-first</id>
|
|
|
|
<phase>process-resources</phase>
|
|
|
|
<goals>
|
|
|
|
<goal>compile</goal>
|
|
|
|
</goals>
|
|
|
|
</execution>
|
|
|
|
<execution>
|
|
|
|
<id>scala-test-compile-first</id>
|
|
|
|
<phase>process-test-resources</phase>
|
|
|
|
<goals>
|
|
|
|
<goal>testCompile</goal>
|
|
|
|
</goals>
|
|
|
|
</execution>
|
|
|
|
<execution>
|
|
|
|
<id>attach-scaladocs</id>
|
|
|
|
<phase>verify</phase>
|
|
|
|
<goals>
|
|
|
|
<goal>doc-jar</goal>
|
|
|
|
</goals>
|
|
|
|
</execution>
|
|
|
|
</executions>
|
|
|
|
<configuration>
|
|
|
|
<scalaVersion>${scala.version}</scalaVersion>
|
|
|
|
<recompileMode>incremental</recompileMode>
|
2013-07-16 14:52:20 -04:00
|
|
|
<useZincServer>true</useZincServer>
|
2012-11-20 18:50:07 -05:00
|
|
|
<args>
|
|
|
|
<arg>-unchecked</arg>
|
2013-08-21 16:50:24 -04:00
|
|
|
<arg>-deprecation</arg>
|
2012-11-20 18:50:07 -05:00
|
|
|
</args>
|
|
|
|
<jvmArgs>
|
|
|
|
<jvmArg>-Xms64m</jvmArg>
|
2013-07-16 14:52:20 -04:00
|
|
|
<jvmArg>-Xms1024m</jvmArg>
|
2012-11-20 18:50:07 -05:00
|
|
|
<jvmArg>-Xmx1024m</jvmArg>
|
2013-06-12 20:20:05 -04:00
|
|
|
<jvmArg>-XX:PermSize=${PermGen}</jvmArg>
|
|
|
|
<jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
|
2012-11-20 18:50:07 -05:00
|
|
|
</jvmArgs>
|
2012-11-27 18:43:30 -05:00
|
|
|
<javacArgs>
|
|
|
|
<javacArg>-source</javacArg>
|
|
|
|
<javacArg>${java.version}</javacArg>
|
|
|
|
<javacArg>-target</javacArg>
|
|
|
|
<javacArg>${java.version}</javacArg>
|
|
|
|
</javacArgs>
|
2012-11-20 18:50:07 -05:00
|
|
|
</configuration>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
|
|
<version>2.5.1</version>
|
|
|
|
<configuration>
|
2012-11-27 18:43:30 -05:00
|
|
|
<source>${java.version}</source>
|
|
|
|
<target>${java.version}</target>
|
2012-11-20 18:50:07 -05:00
|
|
|
<encoding>UTF-8</encoding>
|
|
|
|
<maxmem>1024m</maxmem>
|
|
|
|
</configuration>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
|
|
<version>2.12.4</version>
|
|
|
|
<configuration>
|
|
|
|
<!-- Uses scalatest instead -->
|
|
|
|
<skipTests>true</skipTests>
|
|
|
|
</configuration>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.scalatest</groupId>
|
|
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
|
|
<version>1.0-M2</version>
|
|
|
|
<configuration>
|
|
|
|
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
|
|
|
|
<junitxml>.</junitxml>
|
2013-04-16 07:35:10 -04:00
|
|
|
<filereports>${project.build.directory}/SparkTestSuite.txt</filereports>
|
2013-08-24 02:25:49 -04:00
|
|
|
<argLine>-Xms64m -Xmx3g</argLine>
|
2013-10-05 20:53:11 -04:00
|
|
|
<stderr />
|
2012-11-20 18:50:07 -05:00
|
|
|
</configuration>
|
|
|
|
<executions>
|
|
|
|
<execution>
|
|
|
|
<id>test</id>
|
|
|
|
<goals>
|
|
|
|
<goal>test</goal>
|
|
|
|
</goals>
|
|
|
|
</execution>
|
|
|
|
</executions>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
|
|
<version>2.4</version>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
|
|
<version>1.7</version>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
|
|
<version>2.0</version>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-source-plugin</artifactId>
|
|
|
|
<version>2.2.1</version>
|
|
|
|
<configuration>
|
|
|
|
<attach>true</attach>
|
|
|
|
</configuration>
|
|
|
|
<executions>
|
|
|
|
<execution>
|
|
|
|
<id>create-source-jar</id>
|
|
|
|
<goals>
|
|
|
|
<goal>jar-no-fork</goal>
|
|
|
|
</goals>
|
|
|
|
</execution>
|
|
|
|
</executions>
|
|
|
|
</plugin>
|
|
|
|
</plugins>
|
|
|
|
</pluginManagement>
|
|
|
|
|
|
|
|
<plugins>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-enforcer-plugin</artifactId>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
|
|
<executions>
|
|
|
|
<execution>
|
|
|
|
<id>add-scala-sources</id>
|
|
|
|
<phase>generate-sources</phase>
|
|
|
|
<goals>
|
|
|
|
<goal>add-source</goal>
|
|
|
|
</goals>
|
|
|
|
<configuration>
|
|
|
|
<sources>
|
|
|
|
<source>src/main/scala</source>
|
|
|
|
</sources>
|
|
|
|
</configuration>
|
|
|
|
</execution>
|
|
|
|
<execution>
|
|
|
|
<id>add-scala-test-sources</id>
|
|
|
|
<phase>generate-test-sources</phase>
|
|
|
|
<goals>
|
|
|
|
<goal>add-test-source</goal>
|
|
|
|
</goals>
|
|
|
|
<configuration>
|
|
|
|
<sources>
|
|
|
|
<source>src/test/scala</source>
|
|
|
|
</sources>
|
|
|
|
</configuration>
|
|
|
|
</execution>
|
|
|
|
</executions>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>net.alchim31.maven</groupId>
|
|
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
|
|
</plugin>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
|
|
<artifactId>maven-source-plugin</artifactId>
|
|
|
|
</plugin>
|
|
|
|
</plugins>
|
|
|
|
</build>
|
|
|
|
|
|
|
|
<profiles>
|
2013-04-07 08:17:38 -04:00
|
|
|
<profile>
|
|
|
|
<id>hadoop2-yarn</id>
|
|
|
|
<properties>
|
|
|
|
<hadoop.major.version>2</hadoop.major.version>
|
2013-04-23 17:01:57 -04:00
|
|
|
<!-- 0.23.* is same as 2.0.* - except hardened to run production jobs -->
|
2013-09-13 17:06:45 -04:00
|
|
|
<hadoop.version>0.23.7</hadoop.version>
|
|
|
|
<!--<hadoop.version>2.0.5-alpha</hadoop.version> -->
|
2013-04-07 08:17:38 -04:00
|
|
|
</properties>
|
|
|
|
|
2013-08-15 19:49:24 -04:00
|
|
|
<modules>
|
|
|
|
<module>yarn</module>
|
|
|
|
</modules>
|
|
|
|
|
2013-04-07 08:17:38 -04:00
|
|
|
<repositories>
|
|
|
|
<repository>
|
|
|
|
<id>maven-root</id>
|
|
|
|
<name>Maven root repository</name>
|
|
|
|
<url>http://repo1.maven.org/maven2/</url>
|
|
|
|
<releases>
|
|
|
|
<enabled>true</enabled>
|
|
|
|
</releases>
|
|
|
|
<snapshots>
|
|
|
|
<enabled>false</enabled>
|
|
|
|
</snapshots>
|
|
|
|
</repository>
|
|
|
|
</repositories>
|
|
|
|
|
|
|
|
<dependencyManagement>
|
|
|
|
<dependencies>
|
|
|
|
</dependencies>
|
|
|
|
</dependencyManagement>
|
|
|
|
</profile>
|
2013-07-02 20:03:24 -04:00
|
|
|
<profile>
|
2013-08-27 22:23:54 -04:00
|
|
|
<id>repl-bin</id>
|
2013-07-02 20:03:24 -04:00
|
|
|
<activation>
|
|
|
|
<activeByDefault>false</activeByDefault>
|
|
|
|
</activation>
|
|
|
|
<modules>
|
|
|
|
<module>repl-bin</module>
|
|
|
|
</modules>
|
|
|
|
</profile>
|
2012-11-20 18:50:07 -05:00
|
|
|
</profiles>
|
|
|
|
</project>
|