2012-08-01 16:17:31 -04:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
bin=`dirname "$0"`
|
|
|
|
bin=`cd "$bin"; pwd`
|
|
|
|
|
|
|
|
. "$bin/spark-config.sh"
|
|
|
|
|
2012-08-02 19:12:40 -04:00
|
|
|
if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then
|
|
|
|
. "${SPARK_CONF_DIR}/spark-env.sh"
|
|
|
|
fi
|
|
|
|
|
2012-08-01 16:17:31 -04:00
|
|
|
# Find the port number for the master
|
|
|
|
if [ "$SPARK_MASTER_PORT" = "" ]; then
|
|
|
|
SPARK_MASTER_PORT=7077
|
|
|
|
fi
|
|
|
|
|
2012-10-19 04:08:23 -04:00
|
|
|
if [ "$SPARK_MASTER_IP" = "" ]; then
|
2012-11-28 00:00:04 -05:00
|
|
|
SPARK_MASTER_IP=`hostname`
|
2012-10-19 04:08:23 -04:00
|
|
|
fi
|
|
|
|
|
2012-12-01 21:18:15 -05:00
|
|
|
echo "Master IP: $SPARK_MASTER_IP"
|
2012-08-01 16:17:31 -04:00
|
|
|
|
2012-12-01 21:18:15 -05:00
|
|
|
# Launch the slaves
|
2013-03-06 21:06:32 -05:00
|
|
|
if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
|
|
|
|
exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
|
|
|
|
else
|
|
|
|
if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
|
|
|
|
SPARK_WORKER_WEBUI_PORT=8081
|
|
|
|
fi
|
|
|
|
for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
|
|
|
|
"$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i ))
|
|
|
|
done
|
|
|
|
fi
|