2011-07-14 22:38:25 -04:00
import sbt._
import Keys._
2012-07-30 21:05:13 -04:00
import classpath.ClasspathUtilities.isArchive
import java.io.FileOutputStream
2011-11-09 00:45:29 -05:00
import sbtassembly.Plugin._
import AssemblyKeys._
2011-07-14 22:38:25 -04:00
object SparkBuild extends Build {
2012-03-17 15:31:34 -04:00
// Hadoop version to build against. For example, "0.20.2", "0.20.205.0", or
// "1.0.1" for Apache releases, or "0.20.2-cdh3u3" for Cloudera Hadoop.
val HADOOP_VERSION = "0.20.205.0"
2011-07-14 22:38:25 -04:00
lazy val root = Project ( "root" , file ( "." ) , settings = sharedSettings ) aggregate ( core , repl , examples , bagel )
lazy val core = Project ( "core" , file ( "core" ) , settings = coreSettings )
lazy val repl = Project ( "repl" , file ( "repl" ) , settings = replSettings ) dependsOn ( core )
lazy val examples = Project ( "examples" , file ( "examples" ) , settings = examplesSettings ) dependsOn ( core )
lazy val bagel = Project ( "bagel" , file ( "bagel" ) , settings = bagelSettings ) dependsOn ( core )
def sharedSettings = Defaults . defaultSettings ++ Seq (
organization : = "org.spark-project" ,
2012-06-16 02:55:43 -04:00
version : = "0.6.0-SNAPSHOT" ,
2011-08-31 05:43:05 -04:00
scalaVersion : = "2.9.1" ,
2011-08-02 05:31:55 -04:00
scalacOptions : = Seq ( /* "-deprecation", */ "-unchecked" , "-optimize" ) , // -deprecation is too noisy due to usage of old Hadoop API, enable it once that's no longer an issue
2011-08-30 01:58:53 -04:00
unmanagedJars in Compile <<= baseDirectory map { base => ( base / "lib" ** "*.jar" ) . classpath } ,
2011-07-14 22:38:25 -04:00
retrieveManaged : = true ,
transitiveClassifiers in Scope . GlobalScope : = Seq ( "sources" ) ,
2011-07-20 20:04:29 -04:00
testListeners <<= target . map ( t => Seq ( new eu . henkelmann . sbt . JUnitXmlTestsListener ( t . getAbsolutePath ) ) ) ,
2011-07-31 15:17:12 -04:00
publishTo <<= baseDirectory { base => Some ( Resolver . file ( "Local" , base / "target" / "maven" asFile ) ( Patterns ( true , Resolver . mavenStyleBasePattern ) ) ) } ,
2011-07-14 22:38:25 -04:00
libraryDependencies ++= Seq (
2011-10-17 14:06:41 -04:00
"org.eclipse.jetty" % "jetty-server" % "7.5.3.v20111011" ,
2011-09-26 17:24:46 -04:00
"org.scalatest" %% "scalatest" % "1.6.1" % "test" ,
2012-07-18 20:32:31 -04:00
"org.scalacheck" %% "scalacheck" % "1.9" % "test" ,
"com.novocode" % "junit-interface" % "0.8" % "test"
2011-09-26 19:04:59 -04:00
) ,
2012-06-29 02:51:28 -04:00
parallelExecution : = false ,
2011-09-26 19:04:59 -04:00
/* Workaround for issue #206 (fixed after SBT 0.11.0) */
watchTransitiveSources <<= Defaults . inDependencies [ Task [ Seq [ File ] ] ] ( watchSources . task ,
const ( std . TaskExtra . constant ( Nil ) ) , aggregate = true , includeRoot = true ) apply { _ . join . map ( _ . flatten ) }
2011-07-14 22:38:25 -04:00
)
val slf4jVersion = "1.6.1"
2011-09-26 17:41:07 -04:00
def coreSettings = sharedSettings ++ Seq (
name : = "spark-core" ,
2011-12-15 07:21:14 -05:00
resolvers ++= Seq (
"Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/" ,
2012-03-17 15:31:34 -04:00
"JBoss Repository" at "http://repository.jboss.org/nexus/content/repositories/releases/" ,
2012-06-30 17:45:55 -04:00
"Cloudera Repository" at "http://repository.cloudera.com/artifactory/cloudera-repos/" ,
"Spray Repository" at "http://repo.spray.cc/"
2011-12-15 07:21:14 -05:00
) ,
2011-09-26 17:41:07 -04:00
libraryDependencies ++= Seq (
2012-02-06 18:37:27 -05:00
"com.google.guava" % "guava" % "11.0.1" ,
2011-09-26 17:41:07 -04:00
"log4j" % "log4j" % "1.2.16" ,
"org.slf4j" % "slf4j-api" % slf4jVersion ,
"org.slf4j" % "slf4j-log4j12" % slf4jVersion ,
"com.ning" % "compress-lzf" % "0.8.4" ,
2012-03-17 15:31:34 -04:00
"org.apache.hadoop" % "hadoop-core" % HADOOP_VERSION ,
2011-09-26 17:41:07 -04:00
"asm" % "asm-all" % "3.3.1" ,
2012-03-30 11:38:49 -04:00
"com.google.protobuf" % "protobuf-java" % "2.4.1" ,
2011-12-15 07:21:14 -05:00
"de.javakaffee" % "kryo-serializers" % "0.9" ,
2012-06-29 02:51:28 -04:00
"com.typesafe.akka" % "akka-actor" % "2.0.2" ,
"com.typesafe.akka" % "akka-remote" % "2.0.2" ,
"com.typesafe.akka" % "akka-slf4j" % "2.0.2" ,
2012-02-06 18:37:27 -05:00
"org.jboss.netty" % "netty" % "3.2.6.Final" ,
2012-06-07 03:25:47 -04:00
"it.unimi.dsi" % "fastutil" % "6.4.4" ,
2012-06-30 17:45:55 -04:00
"colt" % "colt" % "1.2.0" ,
"cc.spray" % "spray-can" % "1.0-M2.1" ,
"cc.spray" % "spray-server" % "1.0-M2.1"
2011-09-26 17:41:07 -04:00
)
2012-07-30 21:05:13 -04:00
) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings
2011-09-26 17:41:07 -04:00
def replSettings = sharedSettings ++ Seq (
name : = "spark-repl" ,
libraryDependencies <+= scalaVersion ( "org.scala-lang" % "scala-compiler" % _ )
2012-07-30 21:05:13 -04:00
) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings
2011-09-26 17:41:07 -04:00
def examplesSettings = sharedSettings ++ Seq (
2012-06-07 03:25:47 -04:00
name : = "spark-examples"
2011-09-26 17:41:07 -04:00
)
def bagelSettings = sharedSettings ++ Seq ( name : = "spark-bagel" )
2012-06-22 18:03:09 -04:00
2012-07-30 21:05:13 -04:00
// Fix for "No configuration setting found for key 'akka.version'" exception
// when running Spark from the jar generated by the "assembly" task; see
// http://letitcrash.com/post/21025950392/howto-sbt-assembly-vs-reference-conf
lazy val merge = TaskKey [ File ] ( "merge-reference" ,
"merge all reference.conf" )
lazy val mergeSettings : Seq [ Project . Setting [ _ ] ] = Seq (
merge <<= ( fullClasspath in assembly ) map {
c =>
// collect from all elements of the full classpath
val ( libs , dirs ) =
c map ( _ . data ) partition ( isArchive )
// goal is to simply concatenate files here
val dest = file ( "reference.conf" )
val out = new FileOutputStream ( dest )
val append = IO . transfer ( _ : File , out )
try {
// first collect from managed sources
( dirs * "reference.conf" ) . get foreach append
// then from dependency jars by unzipping and
// collecting reference.conf if present
for ( lib <- libs ) {
IO withTemporaryDirectory {
dir =>
IO . unzip ( lib , dir , "reference.conf" )
( dir * "reference.conf" ) . get foreach append
}
}
// return merged file location as task result
dest
} finally {
out . close ( )
}
} ,
// get rid of the individual files from jars
excludedFiles in assembly <<=
( excludedFiles in assembly ) {
( old ) => ( bases ) =>
old ( bases ) ++ ( bases flatMap ( base =>
( base / "reference.conf" ) . get ) )
} ,
// tell sbt-assembly to include our merged file
assembledMappings in assembly <<=
( assembledMappings in assembly , merge ) map {
( old , merged ) => ( f ) =>
old ( f ) : + ( merged , " reference . conf " )
}
)
2012-06-22 18:03:09 -04:00
def extraAssemblySettings ( ) = Seq ( test in assembly : = { } ) ++ Seq (
mergeStrategy in assembly : = {
case m if m . toLowerCase . endsWith ( "manifest.mf" ) => MergeStrategy . discard
case _ => MergeStrategy . first
}
)
2011-07-14 22:38:25 -04:00
}