378b2315b4
The Maven build was failing on Windows because it tried to call the unix `unzip` utility to extract the Py4J files into core's build directory. I've fixed this issue by using the `maven-antrun-plugin` to perform the unzipping.
I also fixed an issue that prevented tests from running under Windows:
In the Maven ScalaTest plugin, the filename listed in <filereports> is placed under the <reportsDirectory>; the current code places it in a subdirectory of reportsDirectory, e.g.
```
${project.build.directory}/surefire-reports/${project.build.directory}/SparkTestSuite.txt
```
This caused problems under Windows because it would try to create a subdirectory named "c:\\".
Note that the tests still fail under Windows (for other reasons); this PR just allows them to run and fail rather than crash when trying to create the test reports directory.
Author: Josh Rosen <joshrosen@apache.org>
Author: Josh Rosen <rosenville@gmail.com>
Author: Josh Rosen <joshrosen@databricks.com>
Closes #2165 from JoshRosen/windows-support and squashes the following commits:
651d210 [Josh Rosen] Unzip to python/build instead of core/build
fbf3e61 [Josh Rosen] 4 spaces -> 2 spaces
e347668 [Josh Rosen] Fix Maven scalatest filereports path:
4994af1 [Josh Rosen] [SPARK-3061] Use maven-antrun-plugin to unzip Py4J.
375 lines
12 KiB
XML
375 lines
12 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent</artifactId>
|
|
<version>1.1.0-SNAPSHOT</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_2.10</artifactId>
|
|
<properties>
|
|
<sbt.project.name>core</sbt.project.name>
|
|
</properties>
|
|
<packaging>jar</packaging>
|
|
<name>Spark Project Core</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.java.dev.jets3t</groupId>
|
|
<artifactId>jets3t</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-plus</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-security</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
</dependency>
|
|
<!--
|
|
Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
|
|
class exposed in the Java API). The plugin will then remove this dependency from the published
|
|
pom, so that Guava does not pollute the client's compilation classpath.
|
|
-->
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-math3</artifactId>
|
|
<version>3.3</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jcl-over-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.ning</groupId>
|
|
<artifactId>compress-lzf</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.xerial.snappy</groupId>
|
|
<artifactId>snappy-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.jpountz.lz4</groupId>
|
|
<artifactId>lz4</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-net</groupId>
|
|
<artifactId>commons-net</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-remote_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.json4s</groupId>
|
|
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
|
|
<version>3.2.10</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>colt</groupId>
|
|
<artifactId>colt</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.mesos</groupId>
|
|
<artifactId>mesos</artifactId>
|
|
<classifier>${mesos.classifier}</classifier>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-all</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.clearspring.analytics</groupId>
|
|
<artifactId>stream</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-jvm</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-json</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-graphite</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.derby</groupId>
|
|
<artifactId>derby</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.tachyonproject</groupId>
|
|
<artifactId>tachyon-client</artifactId>
|
|
<version>0.5.0</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-jsp</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-webapp</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-servlet</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.powermock</groupId>
|
|
<artifactId>powermock-module-junit4</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.powermock</groupId>
|
|
<artifactId>powermock-api-mockito</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-test</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-all</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalacheck</groupId>
|
|
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.easymock</groupId>
|
|
<artifactId>easymockclassextension</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.novocode</groupId>
|
|
<artifactId>junit-interface</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.spark-project</groupId>
|
|
<artifactId>pyrolite</artifactId>
|
|
<version>2.0.1</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.sf.py4j</groupId>
|
|
<artifactId>py4j</artifactId>
|
|
<version>0.8.2.1</version>
|
|
</dependency>
|
|
</dependencies>
|
|
<build>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
<configuration>
|
|
<environmentVariables>
|
|
<SPARK_HOME>${basedir}/..</SPARK_HOME>
|
|
<SPARK_TESTING>1</SPARK_TESTING>
|
|
<SPARK_CLASSPATH>${spark.classpath}</SPARK_CLASSPATH>
|
|
</environmentVariables>
|
|
</configuration>
|
|
</plugin>
|
|
<!-- Unzip py4j so we can include its files in the jar -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<phase>generate-resources</phase>
|
|
<goals>
|
|
<goal>run</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<tasks>
|
|
<unzip src="../python/lib/py4j-0.8.2.1-src.zip" dest="../python/build" />
|
|
</tasks>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
<configuration>
|
|
<shadedArtifactAttached>false</shadedArtifactAttached>
|
|
<artifactSet>
|
|
<includes>
|
|
<include>com.google.guava:guava</include>
|
|
</includes>
|
|
</artifactSet>
|
|
<filters>
|
|
<!-- See comment in the guava dependency declaration above. -->
|
|
<filter>
|
|
<artifact>com.google.guava:guava</artifact>
|
|
<includes>
|
|
<include>com/google/common/base/Optional*</include>
|
|
</includes>
|
|
</filter>
|
|
</filters>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
</resource>
|
|
<resource>
|
|
<directory>../python</directory>
|
|
<includes>
|
|
<include>pyspark/*.py</include>
|
|
</includes>
|
|
</resource>
|
|
<resource>
|
|
<directory>../python/build</directory>
|
|
<includes>
|
|
<include>py4j/*.py</include>
|
|
</includes>
|
|
</resource>
|
|
</resources>
|
|
</build>
|
|
</project>
|