2011-07-14 22:38:25 -04:00
import sbt._
2012-10-01 03:17:13 -04:00
import sbt.Classpaths.publishTask
2011-07-14 22:38:25 -04:00
import Keys._
2011-11-09 00:45:29 -05:00
import sbtassembly.Plugin._
import AssemblyKeys._
2012-07-31 11:39:24 -04:00
import twirl.sbt.TwirlPlugin._
2012-10-13 18:55:39 -04:00
// For Sonatype publishing
2012-10-14 20:36:20 -04:00
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
2011-07-14 22:38:25 -04:00
object SparkBuild extends Build {
2012-03-17 15:31:34 -04:00
// Hadoop version to build against. For example, "0.20.2", "0.20.205.0", or
2012-10-13 18:55:39 -04:00
// "1.0.3" for Apache releases, or "0.20.2-cdh3u5" for Cloudera Hadoop.
2012-03-17 15:31:34 -04:00
val HADOOP_VERSION = "0.20.205.0"
2012-10-09 18:21:38 -04:00
val HADOOP_MAJOR_VERSION = "1"
2011-07-14 22:38:25 -04:00
2012-10-09 18:21:38 -04:00
// For Hadoop 2 versions such as "2.0.0-mr1-cdh4.1.1", set the HADOOP_MAJOR_VERSION to "2"
//val HADOOP_VERSION = "2.0.0-mr1-cdh4.1.1"
//val HADOOP_MAJOR_VERSION = "2"
2011-07-14 22:38:25 -04:00
2012-10-25 02:21:00 -04:00
lazy val root = Project ( "root" , file ( "." ) , settings = rootSettings ) aggregate ( core , repl , examples , bagel , streaming )
2011-07-14 22:38:25 -04:00
lazy val core = Project ( "core" , file ( "core" ) , settings = coreSettings )
lazy val repl = Project ( "repl" , file ( "repl" ) , settings = replSettings ) dependsOn ( core )
lazy val examples = Project ( "examples" , file ( "examples" ) , settings = examplesSettings ) dependsOn ( core )
lazy val bagel = Project ( "bagel" , file ( "bagel" ) , settings = bagelSettings ) dependsOn ( core )
2012-07-28 23:03:26 -04:00
lazy val streaming = Project ( "streaming" , file ( "streaming" ) , settings = streamingSettings ) dependsOn ( core )
2012-10-01 03:17:13 -04:00
// A configuration to set an alternative publishLocalConfiguration
lazy val MavenCompile = config ( "m2r" ) extend ( Compile )
2012-10-01 18:34:47 -04:00
lazy val publishLocalBoth = TaskKey [ Unit ] ( "publish-local" , "publish local for m2 and ivy" )
2012-10-01 03:17:13 -04:00
2011-07-14 22:38:25 -04:00
def sharedSettings = Defaults . defaultSettings ++ Seq (
organization : = "org.spark-project" ,
2012-10-22 14:49:42 -04:00
version : = "0.7.0-SNAPSHOT" ,
2012-09-24 17:12:48 -04:00
scalaVersion : = "2.9.2" ,
2011-08-02 05:31:55 -04:00
scalacOptions : = Seq ( /* "-deprecation", */ "-unchecked" , "-optimize" ) , // -deprecation is too noisy due to usage of old Hadoop API, enable it once that's no longer an issue
2011-08-30 01:58:53 -04:00
unmanagedJars in Compile <<= baseDirectory map { base => ( base / "lib" ** "*.jar" ) . classpath } ,
2011-07-14 22:38:25 -04:00
retrieveManaged : = true ,
transitiveClassifiers in Scope . GlobalScope : = Seq ( "sources" ) ,
2011-07-20 20:04:29 -04:00
testListeners <<= target . map ( t => Seq ( new eu . henkelmann . sbt . JUnitXmlTestsListener ( t . getAbsolutePath ) ) ) ,
2012-10-05 23:59:31 -04:00
2012-10-13 18:55:39 -04:00
// For Sonatype publishing
2012-10-05 23:59:31 -04:00
resolvers ++= Seq ( "sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots" ,
"sonatype-staging" at "https://oss.sonatype.org/service/local/staging/deploy/maven2/" ) ,
publishMavenStyle : = true ,
2012-10-14 20:36:20 -04:00
//useGpg in Global := true,
2012-10-05 23:59:31 -04:00
pomExtra : = (
< url > http : //spark-project.org/</url>
< licenses >
2012-10-10 21:54:22 -04:00
< license >
< name > BSD License </ name >
< url > https : //github.com/mesos/spark/blob/master/LICENSE</url>
< distribution > repo </ distribution >
</ license >
2012-10-05 23:59:31 -04:00
</ licenses >
< scm >
2012-10-10 21:54:22 -04:00
< connection > scm : git : git @ github . com : mesos / spark . git </ connection >
< url > scm : git : git @ github . com : mesos / spark . git </ url >
2012-10-05 23:59:31 -04:00
</ scm >
< developers >
2012-10-10 21:54:22 -04:00
< developer >
< id > matei </ id >
< name > Matei Zaharia </ name >
< email > matei . zaharia @gmail . com </ email >
< url > http : //www.cs.berkeley.edu/~matei</url>
< organization > U . C . Berkeley Computer Science </ organization >
< organizationUrl > http : //www.cs.berkeley.edu/</organizationUrl>
</ developer >
2012-10-05 23:59:31 -04:00
</ developers >
) ,
2012-10-13 18:55:39 -04:00
2012-10-14 20:36:20 -04:00
/*
2012-10-05 23:59:31 -04:00
publishTo <<= version { ( v : String ) =>
val nexus = "https://oss.sonatype.org/"
2012-10-13 18:55:39 -04:00
if ( v . trim . endsWith ( "SNAPSHOT" ) )
Some ( "sonatype-snapshots" at nexus + "content/repositories/snapshots" )
2012-10-05 23:59:31 -04:00
else
2012-10-10 21:54:22 -04:00
Some ( "sonatype-staging" at nexus + "service/local/staging/deploy/maven2" )
2012-10-05 23:59:31 -04:00
} ,
2012-10-14 20:36:20 -04:00
*/
2012-10-05 23:59:31 -04:00
2011-07-14 22:38:25 -04:00
libraryDependencies ++= Seq (
2011-10-17 14:06:41 -04:00
"org.eclipse.jetty" % "jetty-server" % "7.5.3.v20111011" ,
2011-09-26 17:24:46 -04:00
"org.scalatest" %% "scalatest" % "1.6.1" % "test" ,
2012-07-18 20:32:31 -04:00
"org.scalacheck" %% "scalacheck" % "1.9" % "test" ,
"com.novocode" % "junit-interface" % "0.8" % "test"
2011-09-26 19:04:59 -04:00
) ,
2012-06-29 02:51:28 -04:00
parallelExecution : = false ,
2011-09-26 19:04:59 -04:00
/* Workaround for issue #206 (fixed after SBT 0.11.0) */
watchTransitiveSources <<= Defaults . inDependencies [ Task [ Seq [ File ] ] ] ( watchSources . task ,
2012-10-01 03:17:13 -04:00
const ( std . TaskExtra . constant ( Nil ) ) , aggregate = true , includeRoot = true ) apply { _ . join . map ( _ . flatten ) } ,
otherResolvers : = Seq ( Resolver . file ( "dotM2" , file ( Path . userHome + "/.m2/repository" ) ) ) ,
publishLocalConfiguration in MavenCompile <<= ( packagedArtifacts , deliverLocal , ivyLoggingLevel ) map {
( arts , _ , level ) => new PublishConfiguration ( None , "dotM2" , arts , Seq ( ) , level )
} ,
publishMavenStyle in MavenCompile : = true ,
publishLocal in MavenCompile <<= publishTask ( publishLocalConfiguration in MavenCompile , deliverLocal ) ,
publishLocalBoth <<= Seq ( publishLocal in MavenCompile , publishLocal ) . dependOn
2011-07-14 22:38:25 -04:00
)
val slf4jVersion = "1.6.1"
2011-09-26 17:41:07 -04:00
def coreSettings = sharedSettings ++ Seq (
name : = "spark-core" ,
2011-12-15 07:21:14 -05:00
resolvers ++= Seq (
"Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/" ,
2012-03-17 15:31:34 -04:00
"JBoss Repository" at "http://repository.jboss.org/nexus/content/repositories/releases/" ,
2012-10-02 15:02:00 -04:00
"Spray Repository" at "http://repo.spray.cc/" ,
2012-10-09 18:21:38 -04:00
"Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/"
2011-12-15 07:21:14 -05:00
) ,
2012-10-05 23:59:31 -04:00
2011-09-26 17:41:07 -04:00
libraryDependencies ++= Seq (
2012-02-06 18:37:27 -05:00
"com.google.guava" % "guava" % "11.0.1" ,
2011-09-26 17:41:07 -04:00
"log4j" % "log4j" % "1.2.16" ,
"org.slf4j" % "slf4j-api" % slf4jVersion ,
"org.slf4j" % "slf4j-log4j12" % slf4jVersion ,
"com.ning" % "compress-lzf" % "0.8.4" ,
2012-03-17 15:31:34 -04:00
"org.apache.hadoop" % "hadoop-core" % HADOOP_VERSION ,
2011-09-26 17:41:07 -04:00
"asm" % "asm-all" % "3.3.1" ,
2012-03-30 11:38:49 -04:00
"com.google.protobuf" % "protobuf-java" % "2.4.1" ,
2011-12-15 07:21:14 -05:00
"de.javakaffee" % "kryo-serializers" % "0.9" ,
2012-09-24 17:17:10 -04:00
"com.typesafe.akka" % "akka-actor" % "2.0.3" ,
"com.typesafe.akka" % "akka-remote" % "2.0.3" ,
"com.typesafe.akka" % "akka-slf4j" % "2.0.3" ,
2012-06-07 03:25:47 -04:00
"it.unimi.dsi" % "fastutil" % "6.4.4" ,
2012-06-30 17:45:55 -04:00
"colt" % "colt" % "1.2.0" ,
"cc.spray" % "spray-can" % "1.0-M2.1" ,
2012-10-03 11:58:05 -04:00
"cc.spray" % "spray-server" % "1.0-M2.1" ,
"org.apache.mesos" % "mesos" % "0.9.0-incubating"
2012-10-09 18:21:38 -04:00
) ++ ( if ( HADOOP_MAJOR_VERSION == "2" ) Some ( "org.apache.hadoop" % "hadoop-client" % HADOOP_VERSION ) else None ) . toSeq ,
unmanagedSourceDirectories in Compile <+= baseDirectory { _ / ( "src/hadoop" + HADOOP_MAJOR_VERSION + "/scala" ) }
2012-08-02 17:23:13 -04:00
) ++ assemblySettings ++ extraAssemblySettings ++ Twirl . settings
2011-09-26 17:41:07 -04:00
2012-10-05 23:59:31 -04:00
def rootSettings = sharedSettings ++ Seq (
publish : = { }
)
2011-09-26 17:41:07 -04:00
def replSettings = sharedSettings ++ Seq (
name : = "spark-repl" ,
libraryDependencies <+= scalaVersion ( "org.scala-lang" % "scala-compiler" % _ )
2012-10-02 20:08:16 -04:00
)
2011-09-26 17:41:07 -04:00
def examplesSettings = sharedSettings ++ Seq (
2012-06-07 03:25:47 -04:00
name : = "spark-examples"
2011-09-26 17:41:07 -04:00
)
def bagelSettings = sharedSettings ++ Seq ( name : = "spark-bagel" )
2012-06-22 18:03:09 -04:00
2012-09-02 05:35:17 -04:00
def streamingSettings = sharedSettings ++ Seq (
2012-11-06 12:41:42 -05:00
name : = "spark-streaming" ,
libraryDependencies ++= Seq (
2012-12-05 14:54:42 -05:00
"com.github.sgroschupf" % "zkclient" % "0.1" )
2012-09-02 05:35:17 -04:00
) ++ assemblySettings ++ extraAssemblySettings
2012-07-28 23:03:26 -04:00
2012-06-22 18:03:09 -04:00
def extraAssemblySettings ( ) = Seq ( test in assembly : = { } ) ++ Seq (
2012-10-01 03:17:13 -04:00
mergeStrategy in assembly : = {
case m if m . toLowerCase . endsWith ( "manifest.mf" ) => MergeStrategy . discard
2012-08-02 13:12:00 -04:00
case "reference.conf" => MergeStrategy . concat
2012-06-22 18:03:09 -04:00
case _ => MergeStrategy . first
}
2012-10-01 03:17:13 -04:00
)
2011-07-14 22:38:25 -04:00
}