2013-07-16 20:21:33 -04:00
|
|
|
/*
|
|
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
|
|
* this work for additional information regarding copyright ownership.
|
|
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
* (the "License"); you may not use this file except in compliance with
|
|
|
|
* the License. You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
2013-04-07 08:17:38 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
import sbt._
|
2012-10-01 03:17:13 -04:00
|
|
|
import sbt.Classpaths.publishTask
|
2011-07-14 22:38:25 -04:00
|
|
|
import Keys._
|
2011-11-09 00:45:29 -05:00
|
|
|
import sbtassembly.Plugin._
|
|
|
|
import AssemblyKeys._
|
2014-01-03 04:07:42 -05:00
|
|
|
import scala.util.Properties
|
2014-02-09 13:09:19 -05:00
|
|
|
import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
|
|
|
|
|
2014-03-02 03:35:23 -05:00
|
|
|
import scala.collection.JavaConversions._
|
|
|
|
|
2012-10-13 18:55:39 -04:00
|
|
|
// For Sonatype publishing
|
2012-10-14 20:36:20 -04:00
|
|
|
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
|
2011-07-14 22:38:25 -04:00
|
|
|
|
|
|
|
object SparkBuild extends Build {
|
2013-08-21 20:12:03 -04:00
|
|
|
// Hadoop version to build against. For example, "1.0.4" for Apache releases, or
|
|
|
|
// "2.0.0-mr1-cdh4.2.0" for Cloudera Hadoop. Note that these variables can be set
|
2013-08-24 02:30:17 -04:00
|
|
|
// through the environment variables SPARK_HADOOP_VERSION and SPARK_YARN.
|
2013-08-21 14:38:24 -04:00
|
|
|
val DEFAULT_HADOOP_VERSION = "1.0.4"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-11-23 20:08:01 -05:00
|
|
|
// Whether the Hadoop version to build against is 2.2.x, or a variant of it. This can be set
|
2013-12-01 19:50:24 -05:00
|
|
|
// through the SPARK_IS_NEW_HADOOP environment variable.
|
|
|
|
val DEFAULT_IS_NEW_HADOOP = false
|
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
val DEFAULT_YARN = false
|
2013-04-07 08:17:38 -04:00
|
|
|
|
2013-08-09 16:54:41 -04:00
|
|
|
// HBase version; set as appropriate.
|
2013-08-09 15:45:18 -04:00
|
|
|
val HBASE_VERSION = "0.94.6"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-09-08 17:37:45 -04:00
|
|
|
// Target JVM version
|
2013-10-05 15:41:09 -04:00
|
|
|
val SCALAC_JVM_VERSION = "jvm-1.6"
|
|
|
|
val JAVAC_JVM_VERSION = "1.6"
|
2013-09-08 17:37:45 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
|
|
|
lazy val core = Project("core", file("core"), settings = coreSettings)
|
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val repl = Project("repl", file("repl"), settings = replSettings)
|
2014-01-09 17:31:33 -05:00
|
|
|
.dependsOn(core, graphx, bagel, mllib)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val tools = Project("tools", file("tools"), settings = toolsSettings) dependsOn(core) dependsOn(streaming)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn(core)
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2014-01-09 17:31:33 -05:00
|
|
|
lazy val graphx = Project("graphx", file("graphx"), settings = graphxSettings) dependsOn(core)
|
2013-02-19 15:43:13 -05:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val streaming = Project("streaming", file("streaming"), settings = streamingSettings) dependsOn(core)
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val mllib = Project("mllib", file("mllib"), settings = mllibSettings) dependsOn(core)
|
2013-07-05 14:13:41 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings)
|
2014-03-11 14:16:59 -04:00
|
|
|
.dependsOn(core, graphx, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*) dependsOn(maybeGanglia: _*)
|
2013-07-05 14:13:41 -04:00
|
|
|
|
2013-10-16 20:05:46 -04:00
|
|
|
lazy val assembleDeps = TaskKey[Unit]("assemble-deps", "Build assembly of dependencies and packages Spark projects")
|
2013-10-09 07:24:34 -04:00
|
|
|
|
2012-10-01 03:17:13 -04:00
|
|
|
// A configuration to set an alternative publishLocalConfiguration
|
|
|
|
lazy val MavenCompile = config("m2r") extend(Compile)
|
2012-10-01 18:34:47 -04:00
|
|
|
lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
|
2014-01-03 04:07:42 -05:00
|
|
|
val sparkHome = System.getProperty("user.dir")
|
2014-01-06 05:33:31 -05:00
|
|
|
|
2013-08-14 20:34:34 -04:00
|
|
|
// Allows build configuration to be set through environment variables
|
2014-01-03 04:07:42 -05:00
|
|
|
lazy val hadoopVersion = Properties.envOrElse("SPARK_HADOOP_VERSION", DEFAULT_HADOOP_VERSION)
|
|
|
|
lazy val isNewHadoop = Properties.envOrNone("SPARK_IS_NEW_HADOOP") match {
|
2013-11-23 20:08:01 -05:00
|
|
|
case None => {
|
2013-12-01 19:50:24 -05:00
|
|
|
val isNewHadoopVersion = "2.[2-9]+".r.findFirstIn(hadoopVersion).isDefined
|
|
|
|
(isNewHadoopVersion|| DEFAULT_IS_NEW_HADOOP)
|
2013-11-23 20:08:01 -05:00
|
|
|
}
|
|
|
|
case Some(v) => v.toBoolean
|
|
|
|
}
|
2013-12-12 05:50:10 -05:00
|
|
|
|
2014-01-03 04:07:42 -05:00
|
|
|
lazy val isYarnEnabled = Properties.envOrNone("SPARK_YARN") match {
|
2013-08-24 02:30:17 -04:00
|
|
|
case None => DEFAULT_YARN
|
2013-08-14 20:34:34 -04:00
|
|
|
case Some(v) => v.toBoolean
|
|
|
|
}
|
SPARK-1071: Tidy logging strategy and use of log4j
Prompted by a recent thread on the mailing list, I tried and failed to see if Spark can be made independent of log4j. There are a few cases where control of the underlying logging is pretty useful, and to do that, you have to bind to a specific logger.
Instead I propose some tidying that leaves Spark's use of log4j, but gets rid of warnings and should still enable downstream users to switch. The idea is to pipe everything (except log4j) through SLF4J, and have Spark use SLF4J directly when logging, and where Spark needs to output info (REPL and tests), bind from SLF4J to log4j.
This leaves the same behavior in Spark. It means that downstream users who want to use something except log4j should:
- Exclude dependencies on log4j, slf4j-log4j12 from Spark
- Include dependency on log4j-over-slf4j
- Include dependency on another logger X, and another slf4j-X
- Recreate any log config that Spark does, that is needed, in the other logger's config
That sounds about right.
Here are the key changes:
- Include the jcl-over-slf4j shim everywhere by depending on it in core.
- Exclude dependencies on commons-logging from third-party libraries.
- Include the jul-to-slf4j shim everywhere by depending on it in core.
- Exclude slf4j-* dependencies from third-party libraries to prevent collision or warnings
- Added missing slf4j-log4j12 binding to GraphX, Bagel module tests
And minor/incidental changes:
- Update to SLF4J 1.7.5, which happily matches Hadoop 2’s version and is a recommended update over 1.7.2
- (Remove a duplicate HBase dependency declaration in SparkBuild.scala)
- (Remove a duplicate mockito dependency declaration that was causing warnings and bugging me)
Author: Sean Owen <sowen@cloudera.com>
Closes #570 from srowen/SPARK-1071 and squashes the following commits:
52eac9f [Sean Owen] Add slf4j-over-log4j12 dependency to core (non-test) and remove it from things that depend on core.
77a7fa9 [Sean Owen] SPARK-1071: Tidy logging strategy and use of log4j
2014-02-23 14:40:55 -05:00
|
|
|
lazy val hadoopClient = if (hadoopVersion.startsWith("0.20.") || hadoopVersion == "1.0.0") "hadoop-core" else "hadoop-client"
|
2014-02-27 02:40:49 -05:00
|
|
|
val maybeAvro = if (hadoopVersion.startsWith("0.23.") && isYarnEnabled) Seq("org.apache.avro" % "avro" % "1.7.4") else Seq()
|
2014-03-04 01:31:30 -05:00
|
|
|
|
2014-03-11 14:16:59 -04:00
|
|
|
// Include Ganglia integration if the user has enabled Ganglia
|
|
|
|
// This is isolated from the normal build due to LGPL-licensed code in the library
|
|
|
|
lazy val isGangliaEnabled = Properties.envOrNone("SPARK_GANGLIA_LGPL").isDefined
|
|
|
|
lazy val gangliaProj = Project("spark-ganglia-lgpl", file("extras/spark-ganglia-lgpl"), settings = gangliaSettings).dependsOn(core)
|
|
|
|
val maybeGanglia: Seq[ClasspathDependency] = if (isGangliaEnabled) Seq(gangliaProj) else Seq()
|
|
|
|
val maybeGangliaRef: Seq[ProjectReference] = if (isGangliaEnabled) Seq(gangliaProj) else Seq()
|
|
|
|
|
|
|
|
// Include the Java 8 project if the JVM version is 8+
|
2014-03-04 01:31:30 -05:00
|
|
|
lazy val javaVersion = System.getProperty("java.specification.version")
|
|
|
|
lazy val isJava8Enabled = javaVersion.toDouble >= "1.8".toDouble
|
|
|
|
val maybeJava8Tests = if (isJava8Enabled) Seq[ProjectReference](java8Tests) else Seq[ProjectReference]()
|
|
|
|
lazy val java8Tests = Project("java8-tests", file("extras/java8-tests"), settings = java8TestsSettings).
|
|
|
|
dependsOn(core) dependsOn(streaming % "compile->compile;test->test")
|
|
|
|
|
2014-03-11 14:16:59 -04:00
|
|
|
// Include the YARN project if the user has enabled YARN
|
2014-01-01 22:23:33 -05:00
|
|
|
lazy val yarnAlpha = Project("yarn-alpha", file("yarn/alpha"), settings = yarnAlphaSettings) dependsOn(core)
|
|
|
|
lazy val yarn = Project("yarn", file("yarn/stable"), settings = yarnSettings) dependsOn(core)
|
2013-12-12 05:50:10 -05:00
|
|
|
|
2014-03-11 14:16:59 -04:00
|
|
|
lazy val maybeYarn: Seq[ClasspathDependency] = if (isYarnEnabled) Seq(if (isNewHadoop) yarn else yarnAlpha) else Seq()
|
|
|
|
lazy val maybeYarnRef: Seq[ProjectReference] = if (isYarnEnabled) Seq(if (isNewHadoop) yarn else yarnAlpha) else Seq()
|
2013-08-14 20:34:34 -04:00
|
|
|
|
2014-01-09 19:53:59 -05:00
|
|
|
lazy val externalTwitter = Project("external-twitter", file("external/twitter"), settings = twitterSettings)
|
2013-12-30 14:13:24 -05:00
|
|
|
.dependsOn(streaming % "compile->compile;test->test")
|
|
|
|
|
|
|
|
lazy val externalKafka = Project("external-kafka", file("external/kafka"), settings = kafkaSettings)
|
|
|
|
.dependsOn(streaming % "compile->compile;test->test")
|
|
|
|
|
|
|
|
lazy val externalFlume = Project("external-flume", file("external/flume"), settings = flumeSettings)
|
|
|
|
.dependsOn(streaming % "compile->compile;test->test")
|
2014-01-09 19:53:59 -05:00
|
|
|
|
2013-12-30 14:13:24 -05:00
|
|
|
lazy val externalZeromq = Project("external-zeromq", file("external/zeromq"), settings = zeromqSettings)
|
|
|
|
.dependsOn(streaming % "compile->compile;test->test")
|
2014-01-09 19:53:59 -05:00
|
|
|
|
2013-12-30 14:13:24 -05:00
|
|
|
lazy val externalMqtt = Project("external-mqtt", file("external/mqtt"), settings = mqttSettings)
|
|
|
|
.dependsOn(streaming % "compile->compile;test->test")
|
|
|
|
|
|
|
|
lazy val allExternal = Seq[ClasspathDependency](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt)
|
|
|
|
lazy val allExternalRefs = Seq[ProjectReference](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt)
|
2014-01-09 19:53:59 -05:00
|
|
|
|
2013-12-30 14:13:24 -05:00
|
|
|
lazy val examples = Project("examples", file("examples"), settings = examplesSettings)
|
2014-01-09 17:31:33 -05:00
|
|
|
.dependsOn(core, mllib, graphx, bagel, streaming, externalTwitter) dependsOn(allExternal: _*)
|
2013-12-30 14:13:24 -05:00
|
|
|
|
2014-03-04 01:31:30 -05:00
|
|
|
// Everything except assembly, tools, java8Tests and examples belong to packageProjects
|
2014-03-11 14:16:59 -04:00
|
|
|
lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib, graphx) ++ maybeYarnRef ++ maybeGangliaRef
|
2013-10-09 07:24:34 -04:00
|
|
|
|
2014-03-04 01:31:30 -05:00
|
|
|
lazy val allProjects = packageProjects ++ allExternalRefs ++
|
|
|
|
Seq[ProjectReference](examples, tools, assemblyProj) ++ maybeJava8Tests
|
2013-08-14 20:34:34 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
def sharedSettings = Defaults.defaultSettings ++ Seq(
|
2013-09-06 08:23:01 -04:00
|
|
|
organization := "org.apache.spark",
|
2014-03-02 04:00:16 -05:00
|
|
|
version := "1.0.0-SNAPSHOT",
|
2013-10-05 15:41:09 -04:00
|
|
|
scalaVersion := "2.10.3",
|
2013-09-26 16:58:10 -04:00
|
|
|
scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-deprecation",
|
2013-09-08 17:37:45 -04:00
|
|
|
"-target:" + SCALAC_JVM_VERSION),
|
|
|
|
javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
|
2011-08-30 01:58:53 -04:00
|
|
|
unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },
|
2011-07-14 22:38:25 -04:00
|
|
|
retrieveManaged := true,
|
2014-03-04 01:31:30 -05:00
|
|
|
javaHome := Properties.envOrNone("JAVA_HOME").map(file),
|
2014-02-27 02:40:49 -05:00
|
|
|
// This is to add convenience of enabling sbt -Dsbt.offline=true for making the build offline.
|
|
|
|
offline := "true".equalsIgnoreCase(sys.props("sbt.offline")),
|
2013-01-08 15:44:33 -05:00
|
|
|
retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
|
2011-07-14 22:38:25 -04:00
|
|
|
transitiveClassifiers in Scope.GlobalScope := Seq("sources"),
|
2011-07-20 20:04:29 -04:00
|
|
|
testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))),
|
2012-10-05 23:59:31 -04:00
|
|
|
|
2013-04-19 01:25:21 -04:00
|
|
|
// Fork new JVMs for tests and set Java options for those
|
|
|
|
fork := true,
|
2014-01-06 05:33:31 -05:00
|
|
|
javaOptions in Test += "-Dspark.home=" + sparkHome,
|
|
|
|
javaOptions in Test += "-Dspark.testing=1",
|
2014-03-02 03:35:23 -05:00
|
|
|
javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark").map { case (k,v) => s"-D$k=$v" }.toSeq,
|
2013-08-24 02:25:49 -04:00
|
|
|
javaOptions += "-Xmx3g",
|
2013-12-24 00:20:20 -05:00
|
|
|
// Show full stack trace and duration in test cases.
|
|
|
|
testOptions in Test += Tests.Argument("-oDF"),
|
2014-01-13 22:58:53 -05:00
|
|
|
// Remove certain packages from Scaladoc
|
|
|
|
scalacOptions in (Compile,doc) := Seq("-skip-packages", Seq(
|
|
|
|
"akka",
|
|
|
|
"org.apache.spark.network",
|
|
|
|
"org.apache.spark.deploy",
|
|
|
|
"org.apache.spark.util.collection"
|
|
|
|
).mkString(":")),
|
2013-12-24 00:20:20 -05:00
|
|
|
|
2013-04-11 22:29:37 -04:00
|
|
|
// Only allow one test at a time, even across projects, since they run in the same JVM
|
|
|
|
concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
|
|
|
|
|
2014-03-15 19:43:27 -04:00
|
|
|
resolvers ++= Seq(
|
|
|
|
"Maven Repository" at "https://repo.maven.apache.org/maven2",
|
|
|
|
"Apache Repository" at "https://repository.apache.org/content/repositories/releases",
|
|
|
|
"JBoss Repository" at "https://repository.jboss.org/nexus/content/repositories/releases/",
|
|
|
|
"MQTT Repository" at "https://repo.eclipse.org/content/repositories/paho-releases/",
|
|
|
|
"Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
|
|
|
|
// For Sonatype publishing
|
|
|
|
//"sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
|
|
|
|
//"sonatype-staging" at "https://oss.sonatype.org/service/local/staging/deploy/maven2/",
|
|
|
|
// also check the local Maven repository ~/.m2
|
|
|
|
Resolver.mavenLocal
|
|
|
|
),
|
2012-10-05 23:59:31 -04:00
|
|
|
|
|
|
|
publishMavenStyle := true,
|
|
|
|
|
2012-10-14 20:36:20 -04:00
|
|
|
//useGpg in Global := true,
|
2012-10-05 23:59:31 -04:00
|
|
|
|
|
|
|
pomExtra := (
|
2013-09-02 21:34:03 -04:00
|
|
|
<parent>
|
|
|
|
<groupId>org.apache</groupId>
|
|
|
|
<artifactId>apache</artifactId>
|
|
|
|
<version>13</version>
|
|
|
|
</parent>
|
2014-03-02 04:00:16 -05:00
|
|
|
<url>http://spark.apache.org/</url>
|
2012-10-05 23:59:31 -04:00
|
|
|
<licenses>
|
2012-10-10 21:54:22 -04:00
|
|
|
<license>
|
2013-08-24 02:30:17 -04:00
|
|
|
<name>Apache 2.0 License</name>
|
|
|
|
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
|
2012-10-10 21:54:22 -04:00
|
|
|
<distribution>repo</distribution>
|
|
|
|
</license>
|
2012-10-05 23:59:31 -04:00
|
|
|
</licenses>
|
|
|
|
<scm>
|
2014-03-02 04:00:16 -05:00
|
|
|
<connection>scm:git:git@github.com:apache/spark.git</connection>
|
|
|
|
<url>scm:git:git@github.com:apache/spark.git</url>
|
2012-10-05 23:59:31 -04:00
|
|
|
</scm>
|
|
|
|
<developers>
|
2012-10-10 21:54:22 -04:00
|
|
|
<developer>
|
|
|
|
<id>matei</id>
|
|
|
|
<name>Matei Zaharia</name>
|
|
|
|
<email>matei.zaharia@gmail.com</email>
|
|
|
|
<url>http://www.cs.berkeley.edu/~matei</url>
|
2013-09-01 01:45:53 -04:00
|
|
|
<organization>Apache Software Foundation</organization>
|
2014-03-02 04:00:16 -05:00
|
|
|
<organizationUrl>http://spark.apache.org</organizationUrl>
|
2012-10-10 21:54:22 -04:00
|
|
|
</developer>
|
2012-10-05 23:59:31 -04:00
|
|
|
</developers>
|
2013-08-31 22:27:07 -04:00
|
|
|
<issueManagement>
|
|
|
|
<system>JIRA</system>
|
|
|
|
<url>https://spark-project.atlassian.net/browse/SPARK</url>
|
|
|
|
</issueManagement>
|
2012-10-05 23:59:31 -04:00
|
|
|
),
|
2012-10-13 18:55:39 -04:00
|
|
|
|
2013-12-30 14:13:24 -05:00
|
|
|
/*
|
2012-10-05 23:59:31 -04:00
|
|
|
publishTo <<= version { (v: String) =>
|
|
|
|
val nexus = "https://oss.sonatype.org/"
|
2012-10-13 18:55:39 -04:00
|
|
|
if (v.trim.endsWith("SNAPSHOT"))
|
|
|
|
Some("sonatype-snapshots" at nexus + "content/repositories/snapshots")
|
2012-10-05 23:59:31 -04:00
|
|
|
else
|
2012-10-10 21:54:22 -04:00
|
|
|
Some("sonatype-staging" at nexus + "service/local/staging/deploy/maven2")
|
2012-10-05 23:59:31 -04:00
|
|
|
},
|
|
|
|
|
2013-12-30 14:13:24 -05:00
|
|
|
*/
|
2013-10-07 13:47:45 -04:00
|
|
|
|
2011-07-14 22:38:25 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2014-03-02 03:48:50 -05:00
|
|
|
"io.netty" % "netty-all" % "4.0.17.Final",
|
2014-03-13 15:16:04 -04:00
|
|
|
"org.eclipse.jetty" % "jetty-server" % "9.1.3.v20140225",
|
|
|
|
"org.eclipse.jetty" % "jetty-util" % "9.1.3.v20140225",
|
|
|
|
"org.eclipse.jetty" % "jetty-plus" % "9.1.3.v20140225",
|
|
|
|
"org.eclipse.jetty" % "jetty-security" % "9.1.3.v20140225",
|
2013-11-18 01:37:46 -05:00
|
|
|
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
|
|
|
|
"org.eclipse.jetty.orbit" % "javax.servlet" % "2.5.0.v201103041518" artifacts Artifact("javax.servlet", "jar", "jar"),
|
2013-06-08 04:09:24 -04:00
|
|
|
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
|
|
|
|
"org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
|
2014-01-25 19:32:44 -05:00
|
|
|
"com.novocode" % "junit-interface" % "0.10" % "test",
|
2013-07-15 02:33:48 -04:00
|
|
|
"org.easymock" % "easymock" % "3.1" % "test",
|
2013-11-13 23:44:19 -05:00
|
|
|
"org.mockito" % "mockito-all" % "1.8.5" % "test",
|
2013-07-15 02:33:48 -04:00
|
|
|
"commons-io" % "commons-io" % "2.4" % "test"
|
2013-06-08 04:09:24 -04:00
|
|
|
),
|
|
|
|
|
2014-01-25 19:32:44 -05:00
|
|
|
testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
|
2013-10-24 06:41:40 -04:00
|
|
|
parallelExecution := true,
|
2011-09-26 19:04:59 -04:00
|
|
|
/* Workaround for issue #206 (fixed after SBT 0.11.0) */
|
|
|
|
watchTransitiveSources <<= Defaults.inDependencies[Task[Seq[File]]](watchSources.task,
|
2012-10-01 03:17:13 -04:00
|
|
|
const(std.TaskExtra.constant(Nil)), aggregate = true, includeRoot = true) apply { _.join.map(_.flatten) },
|
|
|
|
|
|
|
|
otherResolvers := Seq(Resolver.file("dotM2", file(Path.userHome + "/.m2/repository"))),
|
|
|
|
publishLocalConfiguration in MavenCompile <<= (packagedArtifacts, deliverLocal, ivyLoggingLevel) map {
|
|
|
|
(arts, _, level) => new PublishConfiguration(None, "dotM2", arts, Seq(), level)
|
|
|
|
},
|
|
|
|
publishMavenStyle in MavenCompile := true,
|
|
|
|
publishLocal in MavenCompile <<= publishTask(publishLocalConfiguration in MavenCompile, deliverLocal),
|
|
|
|
publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
|
2014-02-09 13:09:19 -05:00
|
|
|
) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings
|
2011-07-14 22:38:25 -04:00
|
|
|
|
SPARK-1071: Tidy logging strategy and use of log4j
Prompted by a recent thread on the mailing list, I tried and failed to see if Spark can be made independent of log4j. There are a few cases where control of the underlying logging is pretty useful, and to do that, you have to bind to a specific logger.
Instead I propose some tidying that leaves Spark's use of log4j, but gets rid of warnings and should still enable downstream users to switch. The idea is to pipe everything (except log4j) through SLF4J, and have Spark use SLF4J directly when logging, and where Spark needs to output info (REPL and tests), bind from SLF4J to log4j.
This leaves the same behavior in Spark. It means that downstream users who want to use something except log4j should:
- Exclude dependencies on log4j, slf4j-log4j12 from Spark
- Include dependency on log4j-over-slf4j
- Include dependency on another logger X, and another slf4j-X
- Recreate any log config that Spark does, that is needed, in the other logger's config
That sounds about right.
Here are the key changes:
- Include the jcl-over-slf4j shim everywhere by depending on it in core.
- Exclude dependencies on commons-logging from third-party libraries.
- Include the jul-to-slf4j shim everywhere by depending on it in core.
- Exclude slf4j-* dependencies from third-party libraries to prevent collision or warnings
- Added missing slf4j-log4j12 binding to GraphX, Bagel module tests
And minor/incidental changes:
- Update to SLF4J 1.7.5, which happily matches Hadoop 2’s version and is a recommended update over 1.7.2
- (Remove a duplicate HBase dependency declaration in SparkBuild.scala)
- (Remove a duplicate mockito dependency declaration that was causing warnings and bugging me)
Author: Sean Owen <sowen@cloudera.com>
Closes #570 from srowen/SPARK-1071 and squashes the following commits:
52eac9f [Sean Owen] Add slf4j-over-log4j12 dependency to core (non-test) and remove it from things that depend on core.
77a7fa9 [Sean Owen] SPARK-1071: Tidy logging strategy and use of log4j
2014-02-23 14:40:55 -05:00
|
|
|
val slf4jVersion = "1.7.5"
|
2011-07-14 22:38:25 -04:00
|
|
|
|
2013-06-08 04:09:24 -04:00
|
|
|
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
|
2014-03-09 16:17:07 -04:00
|
|
|
val excludeAsm = ExclusionRule(organization = "org.ow2.asm")
|
|
|
|
val excludeOldAsm = ExclusionRule(organization = "asm")
|
SPARK-1071: Tidy logging strategy and use of log4j
Prompted by a recent thread on the mailing list, I tried and failed to see if Spark can be made independent of log4j. There are a few cases where control of the underlying logging is pretty useful, and to do that, you have to bind to a specific logger.
Instead I propose some tidying that leaves Spark's use of log4j, but gets rid of warnings and should still enable downstream users to switch. The idea is to pipe everything (except log4j) through SLF4J, and have Spark use SLF4J directly when logging, and where Spark needs to output info (REPL and tests), bind from SLF4J to log4j.
This leaves the same behavior in Spark. It means that downstream users who want to use something except log4j should:
- Exclude dependencies on log4j, slf4j-log4j12 from Spark
- Include dependency on log4j-over-slf4j
- Include dependency on another logger X, and another slf4j-X
- Recreate any log config that Spark does, that is needed, in the other logger's config
That sounds about right.
Here are the key changes:
- Include the jcl-over-slf4j shim everywhere by depending on it in core.
- Exclude dependencies on commons-logging from third-party libraries.
- Include the jul-to-slf4j shim everywhere by depending on it in core.
- Exclude slf4j-* dependencies from third-party libraries to prevent collision or warnings
- Added missing slf4j-log4j12 binding to GraphX, Bagel module tests
And minor/incidental changes:
- Update to SLF4J 1.7.5, which happily matches Hadoop 2’s version and is a recommended update over 1.7.2
- (Remove a duplicate HBase dependency declaration in SparkBuild.scala)
- (Remove a duplicate mockito dependency declaration that was causing warnings and bugging me)
Author: Sean Owen <sowen@cloudera.com>
Closes #570 from srowen/SPARK-1071 and squashes the following commits:
52eac9f [Sean Owen] Add slf4j-over-log4j12 dependency to core (non-test) and remove it from things that depend on core.
77a7fa9 [Sean Owen] SPARK-1071: Tidy logging strategy and use of log4j
2014-02-23 14:40:55 -05:00
|
|
|
val excludeCommonsLogging = ExclusionRule(organization = "commons-logging")
|
|
|
|
val excludeSLF4J = ExclusionRule(organization = "org.slf4j")
|
2014-03-02 17:27:53 -05:00
|
|
|
val excludeScalap = ExclusionRule(organization = "org.scala-lang", artifact = "scalap")
|
2013-06-08 04:09:24 -04:00
|
|
|
|
2011-09-26 17:41:07 -04:00
|
|
|
def coreSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-core",
|
2012-10-05 23:59:31 -04:00
|
|
|
|
2011-09-26 17:41:07 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2014-02-27 02:40:49 -05:00
|
|
|
"com.google.guava" % "guava" % "14.0.1",
|
|
|
|
"com.google.code.findbugs" % "jsr305" % "1.3.9",
|
|
|
|
"log4j" % "log4j" % "1.2.17",
|
|
|
|
"org.slf4j" % "slf4j-api" % slf4jVersion,
|
|
|
|
"org.slf4j" % "slf4j-log4j12" % slf4jVersion,
|
|
|
|
"org.slf4j" % "jul-to-slf4j" % slf4jVersion,
|
|
|
|
"org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
|
|
|
|
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
|
|
|
|
"com.ning" % "compress-lzf" % "1.0.0",
|
|
|
|
"org.xerial.snappy" % "snappy-java" % "1.0.5",
|
|
|
|
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
|
|
|
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
|
|
|
|
"org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test",
|
2014-03-02 17:27:53 -05:00
|
|
|
"org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
|
2014-02-27 02:40:49 -05:00
|
|
|
"it.unimi.dsi" % "fastutil" % "6.4.4",
|
|
|
|
"colt" % "colt" % "1.2.0",
|
|
|
|
"org.apache.mesos" % "mesos" % "0.13.0",
|
SPARK-1189: Add Security to Spark - Akka, Http, ConnectionManager, UI use servlets
resubmit pull request. was https://github.com/apache/incubator-spark/pull/332.
Author: Thomas Graves <tgraves@apache.org>
Closes #33 from tgravescs/security-branch-0.9-with-client-rebase and squashes the following commits:
dfe3918 [Thomas Graves] Fix merge conflict since startUserClass now using runAsUser
05eebed [Thomas Graves] Fix dependency lost in upmerge
d1040ec [Thomas Graves] Fix up various imports
05ff5e0 [Thomas Graves] Fix up imports after upmerging to master
ac046b3 [Thomas Graves] Merge remote-tracking branch 'upstream/master' into security-branch-0.9-with-client-rebase
13733e1 [Thomas Graves] Pass securityManager and SparkConf around where we can. Switch to use sparkConf for reading config whereever possible. Added ConnectionManagerSuite unit tests.
4a57acc [Thomas Graves] Change UI createHandler routines to createServlet since they now return servlets
2f77147 [Thomas Graves] Rework from comments
50dd9f2 [Thomas Graves] fix header in SecurityManager
ecbfb65 [Thomas Graves] Fix spacing and formatting
b514bec [Thomas Graves] Fix reference to config
ed3d1c1 [Thomas Graves] Add security.md
6f7ddf3 [Thomas Graves] Convert SaslClient and SaslServer to scala, change spark.authenticate.ui to spark.ui.acls.enable, and fix up various other things from review comments
2d9e23e [Thomas Graves] Merge remote-tracking branch 'upstream/master' into security-branch-0.9-with-client-rebase_rework
5721c5a [Thomas Graves] update AkkaUtilsSuite test for the actorSelection changes, fix typos based on comments, and remove extra lines I missed in rebase from AkkaUtils
f351763 [Thomas Graves] Add Security to Spark - Akka, Http, ConnectionManager, UI to use servlets
2014-03-06 19:27:50 -05:00
|
|
|
"commons-net" % "commons-net" % "2.2",
|
2014-02-27 02:40:49 -05:00
|
|
|
"net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging),
|
|
|
|
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
|
2014-03-09 16:17:07 -04:00
|
|
|
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
|
2014-02-27 02:40:49 -05:00
|
|
|
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
|
|
|
|
"com.codahale.metrics" % "metrics-core" % "3.0.0",
|
|
|
|
"com.codahale.metrics" % "metrics-jvm" % "3.0.0",
|
|
|
|
"com.codahale.metrics" % "metrics-json" % "3.0.0",
|
|
|
|
"com.codahale.metrics" % "metrics-graphite" % "3.0.0",
|
2014-03-09 16:17:07 -04:00
|
|
|
"com.twitter" %% "chill" % "0.3.1" excludeAll(excludeAsm),
|
|
|
|
"com.twitter" % "chill-java" % "0.3.1" excludeAll(excludeAsm),
|
2014-02-27 02:40:49 -05:00
|
|
|
"com.clearspring.analytics" % "stream" % "2.5.1"
|
|
|
|
),
|
|
|
|
libraryDependencies ++= maybeAvro
|
2013-08-24 02:30:17 -04:00
|
|
|
)
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2012-10-05 23:59:31 -04:00
|
|
|
def rootSettings = sharedSettings ++ Seq(
|
|
|
|
publish := {}
|
|
|
|
)
|
|
|
|
|
2013-03-17 01:17:17 -04:00
|
|
|
def replSettings = sharedSettings ++ Seq(
|
2011-09-26 17:41:07 -04:00
|
|
|
name := "spark-repl",
|
2013-09-26 01:57:16 -04:00
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v ),
|
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "jline" % v ),
|
|
|
|
libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v )
|
2013-09-14 13:24:10 -04:00
|
|
|
)
|
2011-09-26 17:41:07 -04:00
|
|
|
|
|
|
|
def examplesSettings = sharedSettings ++ Seq(
|
2013-01-20 22:22:24 -05:00
|
|
|
name := "spark-examples",
|
2013-05-09 19:33:38 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-07-03 02:13:26 -04:00
|
|
|
"com.twitter" %% "algebird-core" % "0.1.11",
|
2014-03-09 16:17:07 -04:00
|
|
|
"org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging),
|
2013-09-26 01:57:16 -04:00
|
|
|
"org.apache.cassandra" % "cassandra-all" % "1.2.6"
|
2013-06-18 08:55:31 -04:00
|
|
|
exclude("com.google.guava", "guava")
|
|
|
|
exclude("com.googlecode.concurrentlinkedhashmap", "concurrentlinkedhashmap-lru")
|
|
|
|
exclude("com.ning","compress-lzf")
|
|
|
|
exclude("io.netty", "netty")
|
|
|
|
exclude("jline","jline")
|
|
|
|
exclude("org.apache.cassandra.deps", "avro")
|
2014-03-02 17:27:53 -05:00
|
|
|
excludeAll(excludeSLF4J)
|
2013-05-09 19:33:38 -04:00
|
|
|
)
|
2013-08-24 02:30:17 -04:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2013-07-18 01:59:00 -04:00
|
|
|
def toolsSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-tools"
|
2013-11-09 16:37:11 -05:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2011-09-26 17:41:07 -04:00
|
|
|
|
2014-01-09 17:31:33 -05:00
|
|
|
def graphxSettings = sharedSettings ++ Seq(
|
2014-01-22 17:40:41 -05:00
|
|
|
name := "spark-graphx",
|
|
|
|
libraryDependencies ++= Seq(
|
2014-01-23 06:54:01 -05:00
|
|
|
"org.jblas" % "jblas" % "1.2.3"
|
2014-01-22 17:40:41 -05:00
|
|
|
)
|
2013-09-18 01:42:12 -04:00
|
|
|
)
|
|
|
|
|
2013-07-30 17:03:15 -04:00
|
|
|
def bagelSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-bagel"
|
2013-07-30 21:31:11 -04:00
|
|
|
)
|
2012-06-22 18:03:09 -04:00
|
|
|
|
2013-07-05 14:38:53 -04:00
|
|
|
def mllibSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-mllib",
|
2013-07-05 14:13:41 -04:00
|
|
|
libraryDependencies ++= Seq(
|
2013-06-21 09:58:01 -04:00
|
|
|
"org.jblas" % "jblas" % "1.2.3"
|
2013-07-05 14:13:41 -04:00
|
|
|
)
|
|
|
|
)
|
|
|
|
|
2012-09-02 05:35:17 -04:00
|
|
|
def streamingSettings = sharedSettings ++ Seq(
|
2012-11-06 12:41:42 -05:00
|
|
|
name := "spark-streaming",
|
|
|
|
libraryDependencies ++= Seq(
|
2014-01-09 19:53:59 -05:00
|
|
|
"commons-io" % "commons-io" % "2.4"
|
2013-01-20 22:22:24 -05:00
|
|
|
)
|
2013-08-24 02:30:17 -04:00
|
|
|
)
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-12-23 00:32:20 -05:00
|
|
|
def yarnCommonSettings = sharedSettings ++ Seq(
|
|
|
|
unmanagedSourceDirectories in Compile <++= baseDirectory { base =>
|
|
|
|
Seq(
|
|
|
|
base / "../common/src/main/scala"
|
|
|
|
)
|
2014-01-01 21:12:06 -05:00
|
|
|
},
|
|
|
|
|
|
|
|
unmanagedSourceDirectories in Test <++= baseDirectory { base =>
|
|
|
|
Seq(
|
|
|
|
base / "../common/src/test/scala"
|
|
|
|
)
|
2013-12-23 00:32:20 -05:00
|
|
|
}
|
2014-01-01 21:12:06 -05:00
|
|
|
|
2013-08-27 22:23:54 -04:00
|
|
|
) ++ extraYarnSettings
|
2013-08-23 13:26:37 -04:00
|
|
|
|
2014-01-01 22:23:33 -05:00
|
|
|
def yarnAlphaSettings = yarnCommonSettings ++ Seq(
|
|
|
|
name := "spark-yarn-alpha"
|
2013-12-04 00:20:27 -05:00
|
|
|
)
|
|
|
|
|
2014-01-01 22:23:33 -05:00
|
|
|
def yarnSettings = yarnCommonSettings ++ Seq(
|
|
|
|
name := "spark-yarn"
|
2013-12-04 00:20:27 -05:00
|
|
|
)
|
|
|
|
|
2014-03-11 14:16:59 -04:00
|
|
|
def gangliaSettings = sharedSettings ++ Seq(
|
|
|
|
name := "spark-ganglia-lgpl",
|
|
|
|
libraryDependencies += "com.codahale.metrics" % "metrics-ganglia" % "3.0.0"
|
|
|
|
)
|
|
|
|
|
2014-03-04 01:31:30 -05:00
|
|
|
def java8TestsSettings = sharedSettings ++ Seq(
|
|
|
|
name := "java8-tests",
|
|
|
|
javacOptions := Seq("-target", "1.8", "-source", "1.8"),
|
|
|
|
testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a")
|
|
|
|
)
|
|
|
|
|
2013-08-23 13:26:37 -04:00
|
|
|
// Conditionally include the YARN dependencies because some tools look at all sub-projects and will complain
|
|
|
|
// if we refer to nonexistent dependencies (e.g. hadoop-yarn-api from a Hadoop version without YARN).
|
|
|
|
def extraYarnSettings = if(isYarnEnabled) yarnEnabledSettings else Seq()
|
|
|
|
|
|
|
|
def yarnEnabledSettings = Seq(
|
2013-07-18 16:36:34 -04:00
|
|
|
libraryDependencies ++= Seq(
|
|
|
|
// Exclude rule required for all ?
|
2014-03-09 16:17:07 -04:00
|
|
|
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm),
|
|
|
|
"org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm)
|
2013-07-18 16:36:34 -04:00
|
|
|
)
|
2013-08-23 13:26:37 -04:00
|
|
|
)
|
2013-07-18 16:36:34 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
def assemblyProjSettings = sharedSettings ++ Seq(
|
2014-01-02 02:37:52 -05:00
|
|
|
libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1",
|
2013-08-24 02:30:17 -04:00
|
|
|
name := "spark-assembly",
|
2013-10-16 20:05:46 -04:00
|
|
|
assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn,
|
2013-10-09 07:24:34 -04:00
|
|
|
jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" },
|
|
|
|
jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" }
|
2012-09-02 05:35:17 -04:00
|
|
|
) ++ assemblySettings ++ extraAssemblySettings
|
2012-07-28 23:03:26 -04:00
|
|
|
|
2013-08-24 02:30:17 -04:00
|
|
|
def extraAssemblySettings() = Seq(
|
|
|
|
test in assembly := {},
|
2012-10-01 03:17:13 -04:00
|
|
|
mergeStrategy in assembly := {
|
|
|
|
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
|
2013-06-25 22:08:19 -04:00
|
|
|
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
|
2013-10-16 01:51:58 -04:00
|
|
|
case "log4j.properties" => MergeStrategy.discard
|
2013-11-12 16:17:48 -05:00
|
|
|
case m if m.toLowerCase.startsWith("meta-inf/services/") => MergeStrategy.filterDistinctLines
|
2012-08-02 13:12:00 -04:00
|
|
|
case "reference.conf" => MergeStrategy.concat
|
2012-06-22 18:03:09 -04:00
|
|
|
case _ => MergeStrategy.first
|
|
|
|
}
|
2012-10-01 03:17:13 -04:00
|
|
|
)
|
2013-12-26 21:02:49 -05:00
|
|
|
|
2013-12-31 03:28:57 -05:00
|
|
|
def twitterSettings() = sharedSettings ++ Seq(
|
2013-12-30 14:13:24 -05:00
|
|
|
name := "spark-streaming-twitter",
|
2013-12-26 21:02:49 -05:00
|
|
|
libraryDependencies ++= Seq(
|
|
|
|
"org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty)
|
|
|
|
)
|
|
|
|
)
|
2014-01-09 19:53:59 -05:00
|
|
|
|
2013-12-31 03:28:57 -05:00
|
|
|
def kafkaSettings() = sharedSettings ++ Seq(
|
2013-12-30 14:13:24 -05:00
|
|
|
name := "spark-streaming-kafka",
|
|
|
|
libraryDependencies ++= Seq(
|
|
|
|
"com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty),
|
2014-01-09 19:53:59 -05:00
|
|
|
"org.apache.kafka" %% "kafka" % "0.8.0"
|
2013-12-30 14:13:24 -05:00
|
|
|
exclude("com.sun.jdmk", "jmxtools")
|
|
|
|
exclude("com.sun.jmx", "jmxri")
|
|
|
|
exclude("net.sf.jopt-simple", "jopt-simple")
|
SPARK-1071: Tidy logging strategy and use of log4j
Prompted by a recent thread on the mailing list, I tried and failed to see if Spark can be made independent of log4j. There are a few cases where control of the underlying logging is pretty useful, and to do that, you have to bind to a specific logger.
Instead I propose some tidying that leaves Spark's use of log4j, but gets rid of warnings and should still enable downstream users to switch. The idea is to pipe everything (except log4j) through SLF4J, and have Spark use SLF4J directly when logging, and where Spark needs to output info (REPL and tests), bind from SLF4J to log4j.
This leaves the same behavior in Spark. It means that downstream users who want to use something except log4j should:
- Exclude dependencies on log4j, slf4j-log4j12 from Spark
- Include dependency on log4j-over-slf4j
- Include dependency on another logger X, and another slf4j-X
- Recreate any log config that Spark does, that is needed, in the other logger's config
That sounds about right.
Here are the key changes:
- Include the jcl-over-slf4j shim everywhere by depending on it in core.
- Exclude dependencies on commons-logging from third-party libraries.
- Include the jul-to-slf4j shim everywhere by depending on it in core.
- Exclude slf4j-* dependencies from third-party libraries to prevent collision or warnings
- Added missing slf4j-log4j12 binding to GraphX, Bagel module tests
And minor/incidental changes:
- Update to SLF4J 1.7.5, which happily matches Hadoop 2’s version and is a recommended update over 1.7.2
- (Remove a duplicate HBase dependency declaration in SparkBuild.scala)
- (Remove a duplicate mockito dependency declaration that was causing warnings and bugging me)
Author: Sean Owen <sowen@cloudera.com>
Closes #570 from srowen/SPARK-1071 and squashes the following commits:
52eac9f [Sean Owen] Add slf4j-over-log4j12 dependency to core (non-test) and remove it from things that depend on core.
77a7fa9 [Sean Owen] SPARK-1071: Tidy logging strategy and use of log4j
2014-02-23 14:40:55 -05:00
|
|
|
excludeAll(excludeNetty, excludeSLF4J)
|
2013-12-30 14:13:24 -05:00
|
|
|
)
|
|
|
|
)
|
2014-01-09 19:53:59 -05:00
|
|
|
|
2013-12-31 03:28:57 -05:00
|
|
|
def flumeSettings() = sharedSettings ++ Seq(
|
2013-12-30 14:13:24 -05:00
|
|
|
name := "spark-streaming-flume",
|
|
|
|
libraryDependencies ++= Seq(
|
2014-03-02 17:27:53 -05:00
|
|
|
"org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty)
|
2013-12-30 14:13:24 -05:00
|
|
|
)
|
|
|
|
)
|
|
|
|
|
2013-12-31 03:28:57 -05:00
|
|
|
def zeromqSettings() = sharedSettings ++ Seq(
|
2013-12-30 14:13:24 -05:00
|
|
|
name := "spark-streaming-zeromq",
|
|
|
|
libraryDependencies ++= Seq(
|
|
|
|
"org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
def mqttSettings() = streamingSettings ++ Seq(
|
|
|
|
name := "spark-streaming-mqtt",
|
2013-12-31 03:28:57 -05:00
|
|
|
libraryDependencies ++= Seq("org.eclipse.paho" % "mqtt-client" % "0.4.0")
|
2013-12-30 14:13:24 -05:00
|
|
|
)
|
2011-07-14 22:38:25 -04:00
|
|
|
}
|