fix paths and change spark to use APP_MEM as application driver memory instead of SPARK_MEM, user should add application jars to SPARK_CLASSPATH
Signed-off-by: shane-huang <shengsheng.huang@intel.com>
This commit is contained in:
parent
1409803763
commit
e8b1ee04fc
41
bin/spark
41
bin/spark
|
@ -31,40 +31,11 @@ if [ -e $FWDIR/conf/spark-env.sh ] ; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -z "$1" ]; then
|
if [ -z "$1" ]; then
|
||||||
echo "Usage: spark-class <class> [<args>]" >&2
|
echo "Usage: spark <class> [<args>]" >&2
|
||||||
|
echo "Usage: export SPARK_CLASSPATH before running the command" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# If this is a standalone cluster daemon, reset SPARK_JAVA_OPTS and SPARK_MEM to reasonable
|
|
||||||
# values for that; it doesn't need a lot
|
|
||||||
if [ "$1" = "org.apache.spark.deploy.master.Master" -o "$1" = "org.apache.spark.deploy.worker.Worker" ]; then
|
|
||||||
SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m}
|
|
||||||
SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.akka.logLifecycleEvents=true"
|
|
||||||
# Do not overwrite SPARK_JAVA_OPTS environment variable in this script
|
|
||||||
OUR_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS" # Empty by default
|
|
||||||
else
|
|
||||||
OUR_JAVA_OPTS="$SPARK_JAVA_OPTS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
# Add java opts for master, worker, executor. The opts maybe null
|
|
||||||
case "$1" in
|
|
||||||
'org.apache.spark.deploy.master.Master')
|
|
||||||
OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_MASTER_OPTS"
|
|
||||||
;;
|
|
||||||
'org.apache.spark.deploy.worker.Worker')
|
|
||||||
OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_WORKER_OPTS"
|
|
||||||
;;
|
|
||||||
'org.apache.spark.executor.StandaloneExecutorBackend')
|
|
||||||
OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS"
|
|
||||||
;;
|
|
||||||
'org.apache.spark.executor.MesosExecutorBackend')
|
|
||||||
OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS"
|
|
||||||
;;
|
|
||||||
'org.apache.spark.repl.Main')
|
|
||||||
OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_REPL_OPTS"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Find the java binary
|
# Find the java binary
|
||||||
if [ -n "${JAVA_HOME}" ]; then
|
if [ -n "${JAVA_HOME}" ]; then
|
||||||
|
@ -78,14 +49,18 @@ else
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Set SPARK_MEM if it isn't already set since we also use it for this process
|
# Set SPARK_MEM if it isn't already set
|
||||||
SPARK_MEM=${SPARK_MEM:-512m}
|
SPARK_MEM=${SPARK_MEM:-512m}
|
||||||
export SPARK_MEM
|
export SPARK_MEM
|
||||||
|
|
||||||
|
# Set APP_MEM if it isn't already set, we use this for this process as the app driver process may need
|
||||||
|
# as much memory as specified in SPARK_MEM
|
||||||
|
APP_MEM=${APP_MEM:-512m}
|
||||||
|
|
||||||
# Set JAVA_OPTS to be able to load native libraries and to set heap size
|
# Set JAVA_OPTS to be able to load native libraries and to set heap size
|
||||||
JAVA_OPTS="$OUR_JAVA_OPTS"
|
JAVA_OPTS="$OUR_JAVA_OPTS"
|
||||||
JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH"
|
JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH"
|
||||||
JAVA_OPTS="$JAVA_OPTS -Xms$SPARK_MEM -Xmx$SPARK_MEM"
|
JAVA_OPTS="$JAVA_OPTS -Xms$APP_MEM -Xmx$APP_MEM"
|
||||||
# Load extra JAVA_OPTS from conf/java-opts, if it exists
|
# Load extra JAVA_OPTS from conf/java-opts, if it exists
|
||||||
if [ -e $FWDIR/conf/java-opts ] ; then
|
if [ -e $FWDIR/conf/java-opts ] ; then
|
||||||
JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`"
|
JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`"
|
||||||
|
|
|
@ -125,7 +125,7 @@ private[spark] class CoarseMesosSchedulerBackend(
|
||||||
StandaloneSchedulerBackend.ACTOR_NAME)
|
StandaloneSchedulerBackend.ACTOR_NAME)
|
||||||
val uri = System.getProperty("spark.executor.uri")
|
val uri = System.getProperty("spark.executor.uri")
|
||||||
if (uri == null) {
|
if (uri == null) {
|
||||||
val runScript = new File(sparkHome, "/sbin/spark-class").getCanonicalPath
|
val runScript = new File(sparkHome, "./sbin/spark-class").getCanonicalPath
|
||||||
command.setValue(
|
command.setValue(
|
||||||
"\"%s\" org.apache.spark.executor.StandaloneExecutorBackend %s %s %s %d".format(
|
"\"%s\" org.apache.spark.executor.StandaloneExecutorBackend %s %s %s %d".format(
|
||||||
runScript, driverUrl, offer.getSlaveId.getValue, offer.getHostname, numCores))
|
runScript, driverUrl, offer.getSlaveId.getValue, offer.getHostname, numCores))
|
||||||
|
|
|
@ -31,7 +31,7 @@ def launch_gateway():
|
||||||
# Launch the Py4j gateway using Spark's run command so that we pick up the
|
# Launch the Py4j gateway using Spark's run command so that we pick up the
|
||||||
# proper classpath and SPARK_MEM settings from spark-env.sh
|
# proper classpath and SPARK_MEM settings from spark-env.sh
|
||||||
on_windows = platform.system() == "Windows"
|
on_windows = platform.system() == "Windows"
|
||||||
script = "/sbin/spark-class.cmd" if on_windows else "/sbin/spark-class"
|
script = "./sbin/spark-class.cmd" if on_windows else "./sbin/spark-class"
|
||||||
command = [os.path.join(SPARK_HOME, script), "py4j.GatewayServer",
|
command = [os.path.join(SPARK_HOME, script), "py4j.GatewayServer",
|
||||||
"--die-on-broken-pipe", "0"]
|
"--die-on-broken-pipe", "0"]
|
||||||
if not on_windows:
|
if not on_windows:
|
||||||
|
|
Loading…
Reference in a new issue