951a5d9398
This PR updates spark-submit to allow submitting Python scripts (currently only with deploy-mode=client, but that's all that was supported before) and updates the PySpark code to properly find various paths, etc. One significant change is that we assume we can always find the Python files either from the Spark assembly JAR (which will happen with the Maven assembly build in make-distribution.sh) or from SPARK_HOME (which will exist in local mode even if you use sbt assembly, and should be enough for testing). This means we no longer need a weird hack to modify the environment for YARN. This patch also updates the Python worker manager to run python with -u, which means unbuffered output (send it to our logs right away instead of waiting a while after stuff was written); this should simplify debugging. In addition, it fixes https://issues.apache.org/jira/browse/SPARK-1709, setting the main class from a JAR's Main-Class attribute if not specified by the user, and fixes a few help strings and style issues in spark-submit. In the future we may want to make the `pyspark` shell use spark-submit as well, but it seems unnecessary for 1.0. Author: Matei Zaharia <matei@databricks.com> Closes #664 from mateiz/py-submit and squashes the following commits: 15e9669 [Matei Zaharia] Fix some uses of path.separator property 051278c [Matei Zaharia] Small style fixes 0afe886 [Matei Zaharia] Add license headers 4650412 [Matei Zaharia] Add pyFiles to PYTHONPATH in executors, remove old YARN stuff, add tests 15f8e1e [Matei Zaharia] Set PYTHONPATH in PythonWorkerFactory in case it wasn't set from outside 47c0655 [Matei Zaharia] More work to make spark-submit work with Python: d4375bd [Matei Zaharia] Clean up description of spark-submit args a bit and add Python ones
345 lines
11 KiB
XML
345 lines
11 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent</artifactId>
|
|
<version>1.0.0-SNAPSHOT</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_2.10</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Spark Project Core</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.java.dev.jets3t</groupId>
|
|
<artifactId>jets3t</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-plus</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-security</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jcl-over-slf4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.ning</groupId>
|
|
<artifactId>compress-lzf</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.xerial.snappy</groupId>
|
|
<artifactId>snappy-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill-java</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-net</groupId>
|
|
<artifactId>commons-net</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-remote_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${akka.group}</groupId>
|
|
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.json4s</groupId>
|
|
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
|
|
<version>3.2.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>colt</groupId>
|
|
<artifactId>colt</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.mesos</groupId>
|
|
<artifactId>mesos</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-all</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.clearspring.analytics</groupId>
|
|
<artifactId>stream</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-jvm</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-json</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.codahale.metrics</groupId>
|
|
<artifactId>metrics-graphite</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.derby</groupId>
|
|
<artifactId>derby</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.tachyonproject</groupId>
|
|
<artifactId>tachyon</artifactId>
|
|
<version>0.4.1-thrift</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-jsp</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-webapp</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-servlet</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.powermock</groupId>
|
|
<artifactId>powermock-module-junit4</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.powermock</groupId>
|
|
<artifactId>powermock-api-mockito</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-test</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-all</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalacheck</groupId>
|
|
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.easymock</groupId>
|
|
<artifactId>easymock</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.novocode</groupId>
|
|
<artifactId>junit-interface</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.spark-project</groupId>
|
|
<artifactId>pyrolite</artifactId>
|
|
<version>2.0.1</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.sf.py4j</groupId>
|
|
<artifactId>py4j</artifactId>
|
|
<version>0.8.1</version>
|
|
</dependency>
|
|
</dependencies>
|
|
<build>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<phase>test</phase>
|
|
<goals>
|
|
<goal>run</goal>
|
|
</goals>
|
|
<configuration>
|
|
<exportAntProperties>true</exportAntProperties>
|
|
<target>
|
|
<property name="spark.classpath" refid="maven.test.classpath" />
|
|
<property environment="env" />
|
|
<fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry.">
|
|
<condition>
|
|
<not>
|
|
<or>
|
|
<isset property="env.SCALA_HOME" />
|
|
<isset property="env.SCALA_LIBRARY_PATH" />
|
|
</or>
|
|
</not>
|
|
</condition>
|
|
</fail>
|
|
</target>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
<configuration>
|
|
<environmentVariables>
|
|
<SPARK_HOME>${basedir}/..</SPARK_HOME>
|
|
<SPARK_TESTING>1</SPARK_TESTING>
|
|
<SPARK_CLASSPATH>${spark.classpath}</SPARK_CLASSPATH>
|
|
</environmentVariables>
|
|
</configuration>
|
|
</plugin>
|
|
<!-- Unzip py4j so we can include its files in the jar -->
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>exec-maven-plugin</artifactId>
|
|
<version>1.2.1</version>
|
|
<executions>
|
|
<execution>
|
|
<phase>generate-resources</phase>
|
|
<goals>
|
|
<goal>exec</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<executable>unzip</executable>
|
|
<workingDirectory>../python</workingDirectory>
|
|
<arguments>
|
|
<argument>-o</argument>
|
|
<argument>lib/py4j*.zip</argument>
|
|
<argument>-d</argument>
|
|
<argument>build</argument>
|
|
</arguments>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
</resource>
|
|
<resource>
|
|
<directory>../python</directory>
|
|
<includes>
|
|
<include>pyspark/*.py</include>
|
|
</includes>
|
|
</resource>
|
|
<resource>
|
|
<directory>../python/build</directory>
|
|
<includes>
|
|
<include>py4j/*.py</include>
|
|
</includes>
|
|
</resource>
|
|
</resources>
|
|
</build>
|
|
</project>
|