2013-07-16 20:21:33 -04:00
|
|
|
/*
|
|
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
|
|
* this work for additional information regarding copyright ownership.
|
|
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
* (the "License"); you may not use this file except in compliance with
|
|
|
|
* the License. You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
2013-04-07 08:17:38 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
import sbt._
|
2012-10-01 03:17:13 -04:00
|
|
|
import sbt.Classpaths.publishTask
|
2011-07-14 22:38:25 -04:00
|
|
|
import Keys._
|
2011-11-09 00:45:29 -05:00
|
|
|
import sbtassembly.Plugin._
|
|
|
|
import AssemblyKeys._
|
2012-10-13 18:55:39 -04:00
|
|
|
// For Sonatype publishing
|
2012-10-14 20:36:20 -04:00
|
|
|
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
|
2011-07-14 22:38:25 -04:00
|
|
|
|
|
|
|
object SparkBuild extends Build {
|
2013-08-21 20:12:03 -04:00
|
|
|
// Hadoop version to build against. For example, "1.0.4" for Apache releases, or
|
|
|
|
// "2.0.0-mr1-cdh4.2.0" for Cloudera Hadoop. Note that these variables can be set
|
2013-08-24 02:30:17 -04:00
|
|
|
// through the environment variables SPARK_HADOOP_VERSION and SPARK_YARN.
|
2013-08-21 14:38:24 -04:00
|
|
|
val DEFAULT_HADOOP_VERSION = "1.0.4"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-11-23 20:08:01 -05:00
|
|
|
// Whether the Hadoop version to build against is 2.2.x, or a variant of it. This can be set
|
2013-12-01 19:50:24 -05:00
|
|
|
// through the SPARK_IS_NEW_HADOOP environment variable.
|
|
|
|
val DEFAULT_IS_NEW_HADOOP = false
|
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
val DEFAULT_YARN = false
|
2013-04-07 08:17:38 -04:00
|
|
|
|
2013-08-09 16:54:41 -04:00
|
|
|
// HBase version; set as appropriate.
|
2013-08-09 15:45:18 -04:00
|
|
|
val HBASE_VERSION = "0.94.6"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-09-08 17:37:45 -04:00
|
|
|
// Target JVM version
|
2013-10-05 15:41:09 -04:00
|
|
|
val SCALAC_JVM_VERSION = "jvm-1.6"
|
|
|
|
val JAVAC_JVM_VERSION = "1.6"
|
2013-09-08 17:37:45 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
|
|
|
lazy val core = Project("core", file("core"), settings = coreSettings)
|
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val repl = Project("repl", file("repl"), settings = replSettings)
|
2013-09-06 13:35:15 -04:00
|
|
|
.dependsOn(core, bagel, mllib)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val examples = Project("examples", file("examples"), settings = examplesSettings)
|
2013-09-06 13:35:15 -04:00
|
|
|
.dependsOn(core, mllib, bagel, streaming)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val tools = Project("tools", file("tools"), settings = toolsSettings) dependsOn(core) dependsOn(streaming)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn(core)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val streaming = Project("streaming", file("streaming"), settings = streamingSettings) dependsOn(core)
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val mllib = Project("mllib", file("mllib"), settings = mllibSettings) dependsOn(core)
|
2013-07-05 14:13:41 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings)
|
|
|
|
.dependsOn(core, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*)
|
2013-07-05 14:13:41 -04:00
|
|
|
|
2013-10-16 20:05:46 -04:00
|
|
|
lazy val assembleDeps = TaskKey[Unit]("assemble-deps", "Build assembly of dependencies and packages Spark projects")
|
2013-10-09 07:24:34 -04:00
|
|
|
|
2012-10-01 03:17:13 -04:00
|
|
|
// A configuration to set an alternative publishLocalConfiguration
|
|
|
|
lazy val MavenCompile = config("m2r") extend(Compile)
|
2012-10-01 18:34:47 -04:00
|
|
|
lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
|
2012-10-01 03:17:13 -04:00
|
|
|
|
2013-08-14 20:34:34 -04:00
|
|
|
// Allows build configuration to be set through environment variables
|
2013-08-16 16:53:16 -04:00
|
|
|
lazy val hadoopVersion = scala.util.Properties.envOrElse("SPARK_HADOOP_VERSION", DEFAULT_HADOOP_VERSION)
|
2013-12-01 19:50:24 -05:00
|
|
|
lazy val isNewHadoop = scala.util.Properties.envOrNone("SPARK_IS_NEW_HADOOP") match {
|
2013-11-23 20:08:01 -05:00
|
|
|
case None => {
|
2013-12-01 19:50:24 -05:00
|
|
|
val isNewHadoopVersion = "2.[2-9]+".r.findFirstIn(hadoopVersion).isDefined
|
|
|
|
(isNewHadoopVersion|| DEFAULT_IS_NEW_HADOOP)
|
2013-11-23 20:08:01 -05:00
|
|
|
}
|
|
|
|
case Some(v) => v.toBoolean
|
|
|
|
}
|
2013-12-12 05:50:10 -05:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val isYarnEnabled = scala.util.Properties.envOrNone("SPARK_YARN") match {
|
|
|
|
case None => DEFAULT_YARN
|
2013-08-14 20:34:34 -04:00
|
|
|
case Some(v) => v.toBoolean
|
|
|
|
}
|
|
|
|
|
|
|
|
// Conditionally include the yarn sub-project
|
2013-12-15 23:30:21 -05:00
|
|
|
lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core)
|
2013-12-12 05:50:10 -05:00
|
|
|
|
2013-12-15 23:30:21 -05:00
|
|
|
//lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core)
|
2013-12-12 05:50:10 -05:00
|
|
|
|
2013-11-23 20:08:01 -05:00
|
|
|
lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]()
|
|
|
|
lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]()
|
2013-08-14 20:34:34 -04:00
|
|
|
|
2013-10-16 20:05:46 -04:00
|
|
|
// Everything except assembly, tools and examples belong to packageProjects
|
2013-10-11 21:23:15 -04:00
|
|
|
lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef
|
2013-10-09 07:24:34 -04:00
|
|
|
|
2013-10-16 20:05:46 -04:00
|
|
|
lazy val allProjects = packageProjects ++ Seq[ProjectReference](examples, tools, assemblyProj)
|
2013-08-14 20:34:34 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
def sharedSettings = Defaults.defaultSettings ++ Seq(
|
2013-09-06 08:23:01 -04:00
|
|
|
organization := "org.apache.spark",
|
2013-10-01 02:27:24 -04:00
|
|
|
version := "0.9.0-incubating-SNAPSHOT",
|
2013-10-05 15:41:09 -04:00
|
|
|
scalaVersion := "2.10.3",
|
2013-09-26 16:58:10 -04:00
|
|
|
scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-deprecation",
|
2013-09-08 17:37:45 -04:00
|
|
|
"-target:" + SCALAC_JVM_VERSION),
|
|
|
|
javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
|
2011-08-30 01:58:53 -04:00
|
|
|
unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },
|
2011-07-14 22:38:25 -04:00
|
|
|
retrieveManaged := true,
|
2013-01-08 15:44:33 -05:00
|
|
|
retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
|
2011-07-14 22:38:25 -04:00
|
|
|
transitiveClassifiers in Scope.GlobalScope := Seq("sources"),
|
2011-07-20 20:04:29 -04:00
|
|
|
testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))),
|
2012-10-05 23:59:31 -04:00
|
|
|
|
2013-04-19 01:25:21 -04:00
|
|
|
// Fork new JVMs for tests and set Java options for those
|
|
|
|
fork := true,
|
2013-08-24 02:25:49 -04:00
|
|
|
javaOptions += "-Xmx3g",
|
2013-04-19 01:25:21 -04:00
|
|
|
|
2013-12-24 00:20:20 -05:00
|
|
|
// Show full stack trace and duration in test cases.
|
|
|
|
testOptions in Test += Tests.Argument("-oDF"),
|
|
|
|
|
2013-04-11 22:29:37 -04:00
|
|
|
// Only allow one test at a time, even across projects, since they run in the same JVM
|
|
|
|
concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
|
|
|
|
|
2013-10-01 18:46:51 -04:00
|
|
|
// also check the local Maven repository ~/.m2
|
|
|
|
resolvers ++= Seq(Resolver.file("Local Maven Repo", file(Path.userHome + "/.m2/repository"))),
|
|
|
|
|
2013-10-16 04:08:37 -04:00
|
|
|
// For Sonatype publishing
|
2012-10-05 23:59:31 -04:00
|
|
|
resolvers ++= Seq("sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
|
|
|
|
"sonatype-staging" at "https://oss.sonatype.org/service/local/staging/deploy/maven2/"),
|
|
|
|
|
|
|
|
publishMavenStyle := true,
|
|
|
|
|
2012-10-14 20:36:20 -04:00
|
|
|
//useGpg in Global := true,
|
2012-10-05 23:59:31 -04:00
|
|
|
|
|
|
|
pomExtra := (
|
2013-09-02 21:34:03 -04:00
|
|
|
<parent>
|
|
|
|
<groupId>org.apache</groupId>
|
|
|
|
<artifactId>apache</artifactId>
|
|
|
|
<version>13</version>
|
|
|
|
</parent>
|
2013-08-31 22:27:07 -04:00
|
|
|
<url>http://spark.incubator.apache.org/</url>
|
2012-10-05 23:59:31 -04:00
|
|
|
<licenses>
|
2012-10-10 21:54:22 -04:00
|
|
|
<license>
|
2013-08-24 02:30:17 -04:00
|
|
|
<name>Apache 2.0 License</name>
|
|
|
|
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
|
2012-10-10 21:54:22 -04:00
|
|
|
<distribution>repo</distribution>
|
|
|
|
</license>
|
2012-10-05 23:59:31 -04:00
|
|
|
</licenses>
|
|
|
|
<scm>
|
2013-08-31 22:27:07 -04:00
|
|
|
<connection>scm:git:git@github.com:apache/incubator-spark.git</connection>
|
|
|
|
<url>scm:git:git@github.com:apache/incubator-spark.git</url>
|
2012-10-05 23:59:31 -04:00
|
|
|
</scm>
|
|
|
|
<developers>
|
2012-10-10 21:54:22 -04:00
|
|
|
<developer>
|
|
|
|
<id>matei</id>
|
|
|
|
<name>Matei Zaharia</name>
|
|
|
|
<email>matei.zaharia@gmail.com</email>
|
|
|
|
<url>http://www.cs.berkeley.edu/~matei</url>
|
2013-09-01 01:45:53 -04:00
|
|
|
<organization>Apache Software Foundation</organization>
|
|
|
|
<organizationUrl>http://spark.incubator.apache.org</organizationUrl>
|
2012-10-10 21:54:22 -04:00
|
|
|
</developer>
|
2012-10-05 23:59:31 -04:00
|
|
|
</developers>
|
2013-08-31 22:27:07 -04:00
|
|
|
<issueManagement>
|
|
|
|
<system>JIRA</system>
|
|
|
|
<url>https://spark-project.atlassian.net/browse/SPARK</url>
|
|
|
|
</issueManagement>
|
2012-10-05 23:59:31 -04:00
|
|
|
),
|
2012-10-13 18:55:39 -04:00
|
|
|
|
2012-10-14 20:36:20 -04:00
|
|
|
/*
|
2012-10-05 23:59:31 -04:00
|
|
|
publishTo <<= version { (v: String) =>
|
|
|
|
val nexus = "https://oss.sonatype.org/"
|
2012-10-13 18:55:39 -04:00
|
|
|
if (v.trim.endsWith("SNAPSHOT"))
|
|
|
|
Some("sonatype-snapshots" at nexus + "content/repositories/snapshots")
|
2012-10-05 23:59:31 -04:00
|
|
|
else
|
2012-10-10 21:54:22 -04:00
|
|
|
Some("sonatype-staging" at nexus + "service/local/staging/deploy/maven2")
|
2012-10-05 23:59:31 -04:00
|
|
|
},
|
|
|
|
|
2012-10-14 20:36:20 -04:00
|
|
|
*/
|
2012-10-05 23:59:31 -04:00
|
|
|
|
2013-10-07 13:47:45 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-09-21 22:50:12 -04:00
|
|
|
"io.netty" % "netty-all" % "4.0.0.CR1",
|
2013-07-03 02:13:26 -04:00
|
|
|
"org.eclipse.jetty" % "jetty-server" % "7.6.8.v20121106",
|
2013-11-18 01:37:46 -05:00
|
|
|
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
|
|
|
|
"org.eclipse.jetty.orbit" % "javax.servlet" % "2.5.0.v201103041518" artifacts Artifact("javax.servlet", "jar", "jar"),
|
2013-06-08 04:09:24 -04:00
|
|
|
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
|
|
|
|
"org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
|
2013-09-06 08:23:01 -04:00
|
|
|
"com.novocode" % "junit-interface" % "0.9" % "test",
|
2013-07-15 02:33:48 -04:00
|
|
|
"org.easymock" % "easymock" % "3.1" % "test",
|
2013-11-13 23:44:19 -05:00
|
|
|
"org.mockito" % "mockito-all" % "1.8.5" % "test",
|
2013-07-15 02:33:48 -04:00
|
|
|
"commons-io" % "commons-io" % "2.4" % "test"
|
2013-06-08 04:09:24 -04:00
|
|
|
),
|
|
|
|
|
2013-10-24 06:41:40 -04:00
|
|
|
parallelExecution := true,
|
2011-09-26 19:04:59 -04:00
|
|
|
/* Workaround for issue #206 (fixed after SBT 0.11.0) */
|
|
|
|
watchTransitiveSources <<= Defaults.inDependencies[Task[Seq[File]]](watchSources.task,
|
2012-10-01 03:17:13 -04:00
|
|
|
const(std.TaskExtra.constant(Nil)), aggregate = true, includeRoot = true) apply { _.join.map(_.flatten) },
|
|
|
|
|
|
|
|
otherResolvers := Seq(Resolver.file("dotM2", file(Path.userHome + "/.m2/repository"))),
|
|
|
|
publishLocalConfiguration in MavenCompile <<= (packagedArtifacts, deliverLocal, ivyLoggingLevel) map {
|
|
|
|
(arts, _, level) => new PublishConfiguration(None, "dotM2", arts, Seq(), level)
|
|
|
|
},
|
|
|
|
publishMavenStyle in MavenCompile := true,
|
|
|
|
publishLocal in MavenCompile <<= publishTask(publishLocalConfiguration in MavenCompile, deliverLocal),
|
|
|
|
publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
|
2013-06-20 12:44:46 -04:00
|
|
|
) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-06-20 12:44:46 -04:00
|
|
|
val slf4jVersion = "1.7.2"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-10-07 13:47:45 -04:00
|
|
|
val excludeCglib = ExclusionRule(organization = "org.sonatype.sisu.inject")
|
2013-06-08 04:09:24 -04:00
|
|
|
val excludeJackson = ExclusionRule(organization = "org.codehaus.jackson")
|
|
|
|
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
|
2013-06-20 12:44:46 -04:00
|
|
|
val excludeAsm = ExclusionRule(organization = "asm")
|
2013-07-30 20:24:45 -04:00
|
|
|
val excludeSnappy = ExclusionRule(organization = "org.xerial.snappy")
|
2013-06-08 04:09:24 -04:00
|
|
|
|
2011-09-26 17:41:07 -04:00
|
|
|
def coreSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-core",
|
2011-12-15 07:21:14 -05:00
|
|
|
resolvers ++= Seq(
|
2013-09-26 01:18:24 -04:00
|
|
|
"JBoss Repository" at "http://repository.jboss.org/nexus/content/repositories/releases/",
|
2013-09-21 22:50:12 -04:00
|
|
|
"Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/"
|
2011-12-15 07:21:14 -05:00
|
|
|
),
|
2012-10-05 23:59:31 -04:00
|
|
|
|
2011-09-26 17:41:07 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-07-12 05:19:16 -04:00
|
|
|
"com.google.guava" % "guava" % "14.0.1",
|
2013-07-15 01:45:55 -04:00
|
|
|
"com.google.code.findbugs" % "jsr305" % "1.3.9",
|
2013-09-15 01:25:12 -04:00
|
|
|
"log4j" % "log4j" % "1.2.17",
|
2013-07-12 05:19:16 -04:00
|
|
|
"org.slf4j" % "slf4j-api" % slf4jVersion,
|
|
|
|
"org.slf4j" % "slf4j-log4j12" % slf4jVersion,
|
2013-11-25 07:40:54 -05:00
|
|
|
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
|
2013-07-12 05:19:16 -04:00
|
|
|
"com.ning" % "compress-lzf" % "0.8.4",
|
2013-09-06 08:23:01 -04:00
|
|
|
"org.xerial.snappy" % "snappy-java" % "1.0.5",
|
2013-07-12 05:19:16 -04:00
|
|
|
"org.ow2.asm" % "asm" % "4.0",
|
2013-12-15 23:30:21 -05:00
|
|
|
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
|
|
|
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
2013-09-21 22:50:12 -04:00
|
|
|
"net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty),
|
2013-07-12 05:19:16 -04:00
|
|
|
"it.unimi.dsi" % "fastutil" % "6.4.4",
|
|
|
|
"colt" % "colt" % "1.2.0",
|
2013-09-26 01:57:16 -04:00
|
|
|
"org.apache.mesos" % "mesos" % "0.13.0",
|
2013-09-15 01:25:12 -04:00
|
|
|
"net.java.dev.jets3t" % "jets3t" % "0.7.1",
|
2013-07-12 05:19:16 -04:00
|
|
|
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
|
2013-11-12 02:14:21 -05:00
|
|
|
"org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
|
2013-09-06 08:23:01 -04:00
|
|
|
"org.apache.avro" % "avro" % "1.7.4",
|
|
|
|
"org.apache.avro" % "avro-ipc" % "1.7.4" excludeAll(excludeNetty),
|
2013-11-14 00:57:02 -05:00
|
|
|
"org.apache.zookeeper" % "zookeeper" % "3.4.5" excludeAll(excludeNetty),
|
2013-09-06 08:23:01 -04:00
|
|
|
"com.codahale.metrics" % "metrics-core" % "3.0.0",
|
|
|
|
"com.codahale.metrics" % "metrics-jvm" % "3.0.0",
|
|
|
|
"com.codahale.metrics" % "metrics-json" % "3.0.0",
|
2013-09-15 01:25:12 -04:00
|
|
|
"com.codahale.metrics" % "metrics-ganglia" % "3.0.0",
|
2013-11-25 07:40:54 -05:00
|
|
|
"com.codahale.metrics" % "metrics-graphite" % "3.0.0",
|
2013-09-14 13:24:10 -04:00
|
|
|
"com.twitter" %% "chill" % "0.3.1",
|
2013-09-26 01:57:16 -04:00
|
|
|
"com.twitter" % "chill-java" % "0.3.1"
|
2013-10-08 01:59:40 -04:00
|
|
|
)
|
2013-08-24 02:30:17 -04:00
|
|
|
)
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2012-10-05 23:59:31 -04:00
|
|
|
def rootSettings = sharedSettings ++ Seq(
|
|
|
|
publish := {}
|
|
|
|
)
|
|
|
|
|
2013-03-17 01:17:17 -04:00
|
|
|
def replSettings = sharedSettings ++ Seq(
|
2011-09-26 17:41:07 -04:00
|
|
|
name := "spark-repl",
|
2013-09-26 01:57:16 -04:00
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v ),
|
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "jline" % v ),
|
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v )
|
2013-09-14 13:24:10 -04:00
|
|
|
)
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2013-12-24 00:20:20 -05:00
|
|
|
|
2011-09-26 17:41:07 -04:00
|
|
|
def examplesSettings = sharedSettings ++ Seq(
|
2013-01-20 22:22:24 -05:00
|
|
|
name := "spark-examples",
|
2013-05-09 19:33:38 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-07-03 02:13:26 -04:00
|
|
|
"com.twitter" %% "algebird-core" % "0.1.11",
|
|
|
|
"org.apache.hbase" % "hbase" % "0.94.6" excludeAll(excludeNetty, excludeAsm),
|
2013-08-09 15:45:18 -04:00
|
|
|
"org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm),
|
2013-09-26 01:57:16 -04:00
|
|
|
"org.apache.cassandra" % "cassandra-all" % "1.2.6"
|
2013-06-18 08:55:31 -04:00
|
|
|
exclude("com.google.guava", "guava")
|
|
|
|
exclude("com.googlecode.concurrentlinkedhashmap", "concurrentlinkedhashmap-lru")
|
|
|
|
exclude("com.ning","compress-lzf")
|
|
|
|
exclude("io.netty", "netty")
|
|
|
|
exclude("jline","jline")
|
|
|
|
exclude("log4j","log4j")
|
|
|
|
exclude("org.apache.cassandra.deps", "avro")
|
2013-07-30 20:24:45 -04:00
|
|
|
excludeAll(excludeSnappy)
|
2013-10-07 13:47:45 -04:00
|
|
|
excludeAll(excludeCglib)
|
2013-05-09 19:33:38 -04:00
|
|
|
)
|
2013-08-24 02:30:17 -04:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2013-07-18 01:59:00 -04:00
|
|
|
def toolsSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-tools"
|
2013-11-09 16:37:11 -05:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2013-07-30 17:03:15 -04:00
|
|
|
def bagelSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-bagel"
|
2013-07-30 21:31:11 -04:00
|
|
|
)
|
2012-06-22 18:03:09 -04:00
|
|
|
|
2013-07-05 14:38:53 -04:00
|
|
|
def mllibSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-mllib",
|
2013-07-05 14:13:41 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-06-21 09:58:01 -04:00
|
|
|
"org.jblas" % "jblas" % "1.2.3"
|
2013-07-05 14:13:41 -04:00
|
|
|
)
|
|
|
|
)
|
|
|
|
|
2012-09-02 05:35:17 -04:00
|
|
|
def streamingSettings = sharedSettings ++ Seq(
|
2012-11-06 12:41:42 -05:00
|
|
|
name := "spark-streaming",
|
2013-03-11 14:37:29 -04:00
|
|
|
resolvers ++= Seq(
|
2013-12-10 21:45:44 -05:00
|
|
|
"Eclipse Repository" at "https://repo.eclipse.org/content/repositories/paho-releases/",
|
2013-10-12 03:02:57 -04:00
|
|
|
"Apache repo" at "https://repository.apache.org/content/repositories/releases"
|
2013-03-11 14:37:29 -04:00
|
|
|
),
|
2013-11-12 02:14:21 -05:00
|
|
|
|
2012-11-06 12:41:42 -05:00
|
|
|
libraryDependencies ++= Seq(
|
2013-12-15 23:30:21 -05:00
|
|
|
"org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy),
|
|
|
|
"com.sksamuel.kafka" %% "kafka" % "0.8.0-beta1"
|
2013-10-12 03:02:57 -04:00
|
|
|
exclude("com.sun.jdmk", "jmxtools")
|
|
|
|
exclude("com.sun.jmx", "jmxri")
|
2013-11-15 02:00:39 -05:00
|
|
|
exclude("net.sf.jopt-simple", "jopt-simple")
|
|
|
|
excludeAll(excludeNetty),
|
2013-12-15 23:30:21 -05:00
|
|
|
"org.eclipse.paho" % "mqtt-client" % "0.4.0",
|
|
|
|
"com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
|
|
|
|
"org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty),
|
|
|
|
"org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
|
2013-01-20 22:22:24 -05:00
|
|
|
)
|
2013-08-24 02:30:17 -04:00
|
|
|
)
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-07-18 16:36:34 -04:00
|
|
|
def yarnSettings = sharedSettings ++ Seq(
|
2013-08-23 13:26:37 -04:00
|
|
|
name := "spark-yarn"
|
2013-08-27 22:23:54 -04:00
|
|
|
) ++ extraYarnSettings
|
2013-08-23 13:26:37 -04:00
|
|
|
|
|
|
|
// Conditionally include the YARN dependencies because some tools look at all sub-projects and will complain
|
|
|
|
// if we refer to nonexistent dependencies (e.g. hadoop-yarn-api from a Hadoop version without YARN).
|
|
|
|
def extraYarnSettings = if(isYarnEnabled) yarnEnabledSettings else Seq()
|
|
|
|
|
|
|
|
def yarnEnabledSettings = Seq(
|
2013-07-18 16:36:34 -04:00
|
|
|
libraryDependencies ++= Seq(
|
|
|
|
// Exclude rule required for all ?
|
2013-10-07 13:47:45 -04:00
|
|
|
"org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib)
|
2013-07-18 16:36:34 -04:00
|
|
|
)
|
2013-08-23 13:26:37 -04:00
|
|
|
)
|
2013-07-18 16:36:34 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
def assemblyProjSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-assembly",
|
2013-10-16 20:05:46 -04:00
|
|
|
assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn,
|
2013-10-09 07:24:34 -04:00
|
|
|
jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" },
|
|
|
|
jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" }
|
2012-09-02 05:35:17 -04:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
def extraAssemblySettings() = Seq(
|
|
|
|
test in assembly := {},
|
2012-10-01 03:17:13 -04:00
|
|
|
mergeStrategy in assembly := {
|
|
|
|
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
|
2013-06-25 22:08:19 -04:00
|
|
|
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
|
2013-10-16 01:51:58 -04:00
|
|
|
case "log4j.properties" => MergeStrategy.discard
|
2013-11-12 16:17:48 -05:00
|
|
|
case m if m.toLowerCase.startsWith("meta-inf/services/") => MergeStrategy.filterDistinctLines
|
2012-08-02 13:12:00 -04:00
|
|
|
case "reference.conf" => MergeStrategy.concat
|
2012-06-22 18:03:09 -04:00
|
|
|
case _ => MergeStrategy.first
|
|
|
|
}
|
2012-10-01 03:17:13 -04:00
|
|
|
)
|
2011-07-14 22:38:25 -04:00
|
|
|
}
|