From fd31adbf27d824f00f62646e13c23f632d1b77d3 Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Sun, 2 Mar 2014 14:27:53 -0800 Subject: [PATCH] SPARK-1084.2 (resubmitted) (Ported from https://github.com/apache/incubator-spark/pull/650 ) This adds one more change though, to fix the scala version warning introduced by json4s recently. Author: Sean Owen Closes #32 from srowen/SPARK-1084.2 and squashes the following commits: 9240abd [Sean Owen] Avoid scala version conflict in scalap induced by json4s dependency 1561cec [Sean Owen] Remove "exclude *" dependencies that are causing Maven warnings, and that are apparently unneeded anyway --- core/pom.xml | 9 +++++++++ examples/pom.xml | 8 -------- external/flume/pom.xml | 4 ---- pom.xml | 37 ++++--------------------------------- project/SparkBuild.scala | 20 +++++++++----------- streaming/pom.xml | 5 ----- 6 files changed, 22 insertions(+), 61 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 6626d7c1b9..2afd250825 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -125,6 +125,15 @@ org.json4s json4s-jackson_${scala.binary.version} 3.2.6 + + + + org.scala-lang + scalap + + it.unimi.dsi diff --git a/examples/pom.xml b/examples/pom.xml index 709907cb7e..d952e2ca4e 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -169,14 +169,6 @@ org.apache.cassandra.deps avro - - org.sonatype.sisu.inject - * - - - org.xerial.snappy - * - diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 95ff872196..bc00ab4573 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -53,10 +53,6 @@ org.jboss.netty netty - - org.xerial.snappy - * - diff --git a/pom.xml b/pom.xml index 39995396b3..7e04efaa9c 100644 --- a/pom.xml +++ b/pom.xml @@ -310,6 +310,9 @@ net.liftweb lift-json_${scala.binary.version} 2.5.1 + org.scala-lang @@ -374,7 +377,7 @@ 3.1 test - + org.mockito mockito-all 1.8.5 @@ -416,14 +419,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - commons-logging commons-logging @@ -448,14 +443,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - @@ -471,14 +458,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - @@ -495,14 +474,6 @@ org.jboss.netty netty - - org.codehaus.jackson - * - - - org.sonatype.sisu.inject - * - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 399886ca1b..d45f6773fa 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -243,13 +243,11 @@ object SparkBuild extends Build { val slf4jVersion = "1.7.5" - val excludeCglib = ExclusionRule(organization = "org.sonatype.sisu.inject") - val excludeJackson = ExclusionRule(organization = "org.codehaus.jackson") val excludeNetty = ExclusionRule(organization = "org.jboss.netty") val excludeAsm = ExclusionRule(organization = "asm") - val excludeSnappy = ExclusionRule(organization = "org.xerial.snappy") val excludeCommonsLogging = ExclusionRule(organization = "commons-logging") val excludeSLF4J = ExclusionRule(organization = "org.slf4j") + val excludeScalap = ExclusionRule(organization = "org.scala-lang", artifact = "scalap") def coreSettings = sharedSettings ++ Seq( name := "spark-core", @@ -273,13 +271,13 @@ object SparkBuild extends Build { "org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), "org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), "org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test", - "org.json4s" %% "json4s-jackson" % "3.2.6", + "org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", "org.apache.mesos" % "mesos" % "0.13.0", "net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging), "org.apache.derby" % "derby" % "10.4.2.0" % "test", - "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib, excludeCommonsLogging, excludeSLF4J), + "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J), "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty), "com.codahale.metrics" % "metrics-core" % "3.0.0", "com.codahale.metrics" % "metrics-jvm" % "3.0.0", @@ -316,7 +314,7 @@ object SparkBuild extends Build { exclude("io.netty", "netty") exclude("jline","jline") exclude("org.apache.cassandra.deps", "avro") - excludeAll(excludeSnappy, excludeCglib, excludeSLF4J) + excludeAll(excludeSLF4J) ) ) ++ assemblySettings ++ extraAssemblySettings @@ -379,10 +377,10 @@ object SparkBuild extends Build { def yarnEnabledSettings = Seq( libraryDependencies ++= Seq( // Exclude rule required for all ? - "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib), - "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty, excludeAsm, excludeCglib) + "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm), + "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm) ) ) @@ -428,7 +426,7 @@ object SparkBuild extends Build { def flumeSettings() = sharedSettings ++ Seq( name := "spark-streaming-flume", libraryDependencies ++= Seq( - "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy) + "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty) ) ) diff --git a/streaming/pom.xml b/streaming/pom.xml index 2f358c9938..1f3366e25f 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -55,11 +55,6 @@ org.eclipse.jetty jetty-server - - org.codehaus.jackson - jackson-mapper-asl - 1.9.11 - org.scala-lang scala-library