HOTFIX: Clean up build in network module.
This is currently breaking the package build for some people (including me). This patch does some general clean-up which also fixes the current issue. - Uses consistent artifact naming - Adds sbt support for this module - Changes tests to use scalatest (fixes the original issue[1]) One thing to note, it turns out that scalatest when invoked in the Maven build doesn't succesfully detect JUnit Java tests. This is a long standing issue, I noticed it applies to all of our current test suites as well. I've created SPARK-4159 to fix this. [1] The original issue is that we need to allocate extra memory for the tests, happens by default in our scalatest configuration. Author: Patrick Wendell <pwendell@gmail.com> Closes #3025 from pwendell/hotfix and squashes the following commits: faa9053 [Patrick Wendell] HOTFIX: Clean up build in network module.
This commit is contained in:
parent
26d31d15fd
commit
0734d09320
|
@ -46,7 +46,7 @@
|
|||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>network</artifactId>
|
||||
<artifactId>spark-network-common_2.10</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
|
@ -27,12 +27,12 @@
|
|||
</parent>
|
||||
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>network</artifactId>
|
||||
<artifactId>spark-network-common_2.10</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>Shuffle Streaming Service</name>
|
||||
<name>Spark Project Common Network Code</name>
|
||||
<url>http://spark.apache.org/</url>
|
||||
<properties>
|
||||
<sbt.project.name>network</sbt.project.name>
|
||||
<sbt.project.name>network-common</sbt.project.name>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -59,6 +59,11 @@
|
|||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.novocode</groupId>
|
||||
<artifactId>junit-interface</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
|
@ -69,25 +74,20 @@
|
|||
<artifactId>mockito-all</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<outputDirectory>target/java/classes</outputDirectory>
|
||||
<testOutputDirectory>target/java/test-classes</testOutputDirectory>
|
||||
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
||||
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.17</version>
|
||||
<configuration>
|
||||
<skipTests>false</skipTests>
|
||||
<includes>
|
||||
<include>**/Test*.java</include>
|
||||
<include>**/*Test.java</include>
|
||||
<include>**/*Suite.java</include>
|
||||
</includes>
|
||||
</configuration>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
|
|
@ -31,10 +31,10 @@ object BuildCommons {
|
|||
private val buildLocation = file(".").getAbsoluteFile.getParentFile
|
||||
|
||||
val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl,
|
||||
sql, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt,
|
||||
sql, networkCommon, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt,
|
||||
streamingTwitter, streamingZeromq) =
|
||||
Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl",
|
||||
"sql", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka",
|
||||
"sql", "network-common", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka",
|
||||
"streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _))
|
||||
|
||||
val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, sparkGangliaLgpl, sparkKinesisAsl) =
|
||||
|
@ -142,7 +142,9 @@ object SparkBuild extends PomBuild {
|
|||
|
||||
// TODO: Add Sql to mima checks
|
||||
allProjects.filterNot(x => Seq(spark, sql, hive, hiveThriftServer, catalyst, repl,
|
||||
streamingFlumeSink).contains(x)).foreach(x => enable(MimaBuild.mimaSettings(sparkHome, x))(x))
|
||||
streamingFlumeSink, networkCommon).contains(x)).foreach {
|
||||
x => enable(MimaBuild.mimaSettings(sparkHome, x))(x)
|
||||
}
|
||||
|
||||
/* Enable Assembly for all assembly projects */
|
||||
assemblyProjects.foreach(enable(Assembly.settings))
|
||||
|
|
Loading…
Reference in a new issue