Merge remote-tracking branch 'kalpit/master'
Conflicts: project/SparkBuild.scala
This commit is contained in:
Matei Zaharia
commit
1cb3eb9762
|
|
@ -30,7 +30,7 @@
|
|||
# SPARK_NICENESS The scheduling priority for daemons. Defaults to 0.
|
||||
##
|
||||
|
||||
usage="Usage: spark-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <spark-command> <args...>"
|
||||
usage="Usage: spark-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <spark-command> <spark-instance-number> <args...>"
|
||||
|
||||
# if no args specified, show usage
|
||||
if [ $# -le 1 ]; then
|
||||
|
|
@ -48,6 +48,8 @@ startStop=$1
|
|||
shift
|
||||
command=$1
|
||||
shift
|
||||
instance=$1
|
||||
shift
|
||||
|
||||
spark_rotate_log ()
|
||||
{
|
||||
|
|
@ -92,10 +94,10 @@ if [ "$SPARK_PID_DIR" = "" ]; then
|
|||
fi
|
||||
|
||||
# some variables
|
||||
export SPARK_LOGFILE=spark-$SPARK_IDENT_STRING-$command-$HOSTNAME.log
|
||||
export SPARK_LOGFILE=spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.log
|
||||
export SPARK_ROOT_LOGGER="INFO,DRFA"
|
||||
log=$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$HOSTNAME.out
|
||||
pid=$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command.pid
|
||||
log=$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.out
|
||||
pid=$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command-$instance.pid
|
||||
|
||||
# Set default scheduling priority
|
||||
if [ "$SPARK_NICENESS" = "" ]; then
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Run a Spark command on all slave hosts.
|
||||
|
||||
usage="Usage: spark-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..."
|
||||
usage="Usage: spark-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command instance-number args..."
|
||||
|
||||
# if no args specified, show usage
|
||||
if [ $# -le 1 ]; then
|
||||
|
|
|
|||
|
|
@ -32,4 +32,4 @@ if [ "$SPARK_PUBLIC_DNS" = "" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
"$bin"/spark-daemon.sh start spark.deploy.master.Master --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT
|
||||
"$bin"/spark-daemon.sh start spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT
|
||||
|
|
|
|||
|
|
@ -11,4 +11,4 @@ if [ "$SPARK_PUBLIC_DNS" = "" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
"$bin"/spark-daemon.sh start spark.deploy.worker.Worker $1
|
||||
"$bin"/spark-daemon.sh start spark.deploy.worker.Worker "$@"
|
||||
|
|
|
|||
|
|
@ -21,4 +21,13 @@ fi
|
|||
echo "Master IP: $SPARK_MASTER_IP"
|
||||
|
||||
# Launch the slaves
|
||||
exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
|
||||
if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
|
||||
exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
|
||||
else
|
||||
if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
|
||||
SPARK_WORKER_WEBUI_PORT=8081
|
||||
fi
|
||||
for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
|
||||
"$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i ))
|
||||
done
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@ bin=`cd "$bin"; pwd`
|
|||
|
||||
. "$bin/spark-config.sh"
|
||||
|
||||
"$bin"/spark-daemon.sh stop spark.deploy.master.Master
|
||||
"$bin"/spark-daemon.sh stop spark.deploy.master.Master 1
|
||||
|
|
|
|||
|
|
@ -7,4 +7,14 @@ bin=`cd "$bin"; pwd`
|
|||
|
||||
. "$bin/spark-config.sh"
|
||||
|
||||
"$bin"/spark-daemons.sh stop spark.deploy.worker.Worker
|
||||
if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then
|
||||
. "${SPARK_CONF_DIR}/spark-env.sh"
|
||||
fi
|
||||
|
||||
if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
|
||||
"$bin"/spark-daemons.sh stop spark.deploy.worker.Worker 1
|
||||
else
|
||||
for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
|
||||
"$bin"/spark-daemons.sh stop spark.deploy.worker.Worker $(( $i + 1 ))
|
||||
done
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
|
||||
# - SPARK_WORKER_MEMORY, to set how much memory to use (e.g. 1000m, 2g)
|
||||
# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT
|
||||
# - SPARK_WORKER_INSTANCES, to set the number of worker instances/processes to be spawned on every slave machine
|
||||
#
|
||||
# Finally, Spark also relies on the following variables, but these can be set
|
||||
# on just the *master* (i.e. in your driver program), and will automatically
|
||||
|
|
|
|||
|
|
@ -92,10 +92,10 @@ object SparkBuild extends Build {
|
|||
*/
|
||||
|
||||
libraryDependencies ++= Seq(
|
||||
"org.eclipse.jetty" % "jetty-server" % "7.5.3.v20111011",
|
||||
"org.eclipse.jetty" % "jetty-server" % "7.6.8.v20121106",
|
||||
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
|
||||
"org.scalacheck" %% "scalacheck" % "1.10.0" % "test",
|
||||
"com.novocode" % "junit-interface" % "0.8" % "test",
|
||||
"com.novocode" % "junit-interface" % "0.9" % "test",
|
||||
"org.easymock" % "easymock" % "3.1" % "test"
|
||||
),
|
||||
parallelExecution := false,
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
sbt.version=0.11.3
|
||||
sbt.version=0.12.2
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/release
|
|||
|
||||
resolvers += "Spray Repository" at "http://repo.spray.cc/"
|
||||
|
||||
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.3")
|
||||
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.5")
|
||||
|
||||
addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.1.0-RC1")
|
||||
addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.1.1")
|
||||
|
||||
addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.0.0")
|
||||
addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.2.0")
|
||||
|
||||
addSbtPlugin("io.spray" %% "sbt-twirl" % "0.6.1")
|
||||
|
||||
|
|
|
|||
2
run
2
run
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
SCALA_VERSION=2.9.2
|
||||
SCALA_VERSION=2.9.3
|
||||
|
||||
# Figure out where the Scala framework is installed
|
||||
FWDIR="$(cd `dirname $0`; pwd)"
|
||||
|
|
|
|||
Loading…
Reference in a new issue