diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala index 19a039e330..cf60d14f03 100644 --- a/core/src/main/scala/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/spark/api/python/PythonRDD.scala @@ -38,7 +38,7 @@ private[spark] class PythonRDD[T: ClassManifest]( override def compute(split: Split, context: TaskContext): Iterator[Array[Byte]] = { val SPARK_HOME = new ProcessBuilder().environment().get("SPARK_HOME") - val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/pyspark/pyspark/worker.py")) + val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/python/pyspark/worker.py")) // Add the environmental variables to the process. val currentEnvVars = pb.environment() diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 577f3ebe70..c9ce589c1b 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -30,8 +30,8 @@ if ENV['SKIP_SCALADOC'] != '1' end if ENV['SKIP_EPYDOC'] != '1' - puts "Moving to pyspark directory and building epydoc." - cd("../pyspark") + puts "Moving to python directory and building epydoc." + cd("../python") puts `epydoc --config epydoc.conf` puts "Moving back into docs dir." @@ -40,8 +40,8 @@ if ENV['SKIP_EPYDOC'] != '1' puts "echo making directory pyspark" mkdir_p "pyspark" - puts "cp -r ../pyspark/docs/. api/pyspark" - cp_r("../pyspark/docs/.", "api/pyspark") + puts "cp -r ../python/docs/. api/pyspark" + cp_r("../python/docs/.", "api/pyspark") cd("..") end diff --git a/pyspark-shell b/pyspark-shell new file mode 100755 index 0000000000..27aaac3a26 --- /dev/null +++ b/pyspark-shell @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +FWDIR="`dirname $0`" +exec $FWDIR/run-pyspark $FWDIR/python/pyspark/shell.py "$@" diff --git a/pyspark/pyspark-shell b/pyspark/pyspark-shell deleted file mode 100755 index e3736826e8..0000000000 --- a/pyspark/pyspark-shell +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -FWDIR="`dirname $0`" -exec $FWDIR/run-pyspark $FWDIR/pyspark/shell.py "$@" diff --git a/pyspark/.gitignore b/python/.gitignore similarity index 100% rename from pyspark/.gitignore rename to python/.gitignore diff --git a/pyspark/epydoc.conf b/python/epydoc.conf similarity index 100% rename from pyspark/epydoc.conf rename to python/epydoc.conf diff --git a/pyspark/examples/kmeans.py b/python/examples/kmeans.py similarity index 100% rename from pyspark/examples/kmeans.py rename to python/examples/kmeans.py diff --git a/pyspark/examples/logistic_regression.py b/python/examples/logistic_regression.py similarity index 100% rename from pyspark/examples/logistic_regression.py rename to python/examples/logistic_regression.py diff --git a/pyspark/examples/pi.py b/python/examples/pi.py similarity index 100% rename from pyspark/examples/pi.py rename to python/examples/pi.py diff --git a/pyspark/examples/transitive_closure.py b/python/examples/transitive_closure.py similarity index 100% rename from pyspark/examples/transitive_closure.py rename to python/examples/transitive_closure.py diff --git a/pyspark/examples/wordcount.py b/python/examples/wordcount.py similarity index 100% rename from pyspark/examples/wordcount.py rename to python/examples/wordcount.py diff --git a/pyspark/lib/PY4J_LICENSE.txt b/python/lib/PY4J_LICENSE.txt similarity index 100% rename from pyspark/lib/PY4J_LICENSE.txt rename to python/lib/PY4J_LICENSE.txt diff --git a/pyspark/lib/PY4J_VERSION.txt b/python/lib/PY4J_VERSION.txt similarity index 100% rename from pyspark/lib/PY4J_VERSION.txt rename to python/lib/PY4J_VERSION.txt diff --git a/pyspark/lib/py4j0.7.egg b/python/lib/py4j0.7.egg similarity index 100% rename from pyspark/lib/py4j0.7.egg rename to python/lib/py4j0.7.egg diff --git a/pyspark/lib/py4j0.7.jar b/python/lib/py4j0.7.jar similarity index 100% rename from pyspark/lib/py4j0.7.jar rename to python/lib/py4j0.7.jar diff --git a/pyspark/pyspark/__init__.py b/python/pyspark/__init__.py similarity index 82% rename from pyspark/pyspark/__init__.py rename to python/pyspark/__init__.py index 1ab360a666..c595ae0842 100644 --- a/pyspark/pyspark/__init__.py +++ b/python/pyspark/__init__.py @@ -10,7 +10,7 @@ Public classes: """ import sys import os -sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "pyspark/lib/py4j0.7.egg")) +sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "python/lib/py4j0.7.egg")) from pyspark.context import SparkContext diff --git a/pyspark/pyspark/broadcast.py b/python/pyspark/broadcast.py similarity index 100% rename from pyspark/pyspark/broadcast.py rename to python/pyspark/broadcast.py diff --git a/pyspark/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py similarity index 100% rename from pyspark/pyspark/cloudpickle.py rename to python/pyspark/cloudpickle.py diff --git a/pyspark/pyspark/context.py b/python/pyspark/context.py similarity index 100% rename from pyspark/pyspark/context.py rename to python/pyspark/context.py diff --git a/pyspark/pyspark/java_gateway.py b/python/pyspark/java_gateway.py similarity index 100% rename from pyspark/pyspark/java_gateway.py rename to python/pyspark/java_gateway.py diff --git a/pyspark/pyspark/join.py b/python/pyspark/join.py similarity index 100% rename from pyspark/pyspark/join.py rename to python/pyspark/join.py diff --git a/pyspark/pyspark/rdd.py b/python/pyspark/rdd.py similarity index 100% rename from pyspark/pyspark/rdd.py rename to python/pyspark/rdd.py diff --git a/pyspark/pyspark/serializers.py b/python/pyspark/serializers.py similarity index 100% rename from pyspark/pyspark/serializers.py rename to python/pyspark/serializers.py diff --git a/pyspark/pyspark/shell.py b/python/pyspark/shell.py similarity index 100% rename from pyspark/pyspark/shell.py rename to python/pyspark/shell.py diff --git a/pyspark/pyspark/worker.py b/python/pyspark/worker.py similarity index 100% rename from pyspark/pyspark/worker.py rename to python/pyspark/worker.py diff --git a/run b/run index ed788c4db3..08e2b2434b 100755 --- a/run +++ b/run @@ -63,7 +63,7 @@ CORE_DIR="$FWDIR/core" REPL_DIR="$FWDIR/repl" EXAMPLES_DIR="$FWDIR/examples" BAGEL_DIR="$FWDIR/bagel" -PYSPARK_DIR="$FWDIR/pyspark" +PYSPARK_DIR="$FWDIR/python" # Build up classpath CLASSPATH="$SPARK_CLASSPATH" diff --git a/pyspark/run-pyspark b/run-pyspark similarity index 86% rename from pyspark/run-pyspark rename to run-pyspark index 4d10fbea8b..deb0d708b3 100755 --- a/pyspark/run-pyspark +++ b/run-pyspark @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; cd ../; pwd)" +FWDIR="$(cd `dirname $0`; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" @@ -18,7 +18,7 @@ fi export PYSPARK_PYTHON # Add the PySpark classes to the Python path: -export PYTHONPATH=$SPARK_HOME/pyspark/:$PYTHONPATH +export PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH # Launch with `scala` by default: if [[ "$SPARK_LAUNCH_WITH_SCALA" != "0" ]] ; then diff --git a/run2.cmd b/run2.cmd index 9c50804e69..83464b1166 100644 --- a/run2.cmd +++ b/run2.cmd @@ -34,7 +34,7 @@ set CORE_DIR=%FWDIR%core set REPL_DIR=%FWDIR%repl set EXAMPLES_DIR=%FWDIR%examples set BAGEL_DIR=%FWDIR%bagel -set PYSPARK_DIR=%FWDIR%pyspark +set PYSPARK_DIR=%FWDIR%python rem Build up classpath set CLASSPATH=%SPARK_CLASSPATH%;%MESOS_CLASSPATH%;%FWDIR%conf;%CORE_DIR%\target\scala-%SCALA_VERSION%\classes