[SPARK-11653][DEPLOY] Allow spark-daemon.sh to run in the foreground
## What changes were proposed in this pull request? Add a SPARK_NO_DAEMONIZE environment variable flag to spark-daemon.sh that causes the process it would run to be run in the foreground. It looks like there has been some prior work in https://github.com/apache/spark/pull/3881, but there was some talk about these being refactored. I'm not sure if that happened or not, but that PR is almost 2 years old at this point so it was worth revisiting. ## How was this patch tested? ./dev/run-tests still seems to work. It doesn't look like these scripts have tests, but if I missed them just let me know. Author: Mike Ihbe <mikejihbe@gmail.com> Closes #15338 from mikejihbe/SPARK-11653.
This commit is contained in:
parent
4bd17c4606
commit
c2c107abad
|
@ -63,3 +63,4 @@
|
|||
# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp)
|
||||
# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER)
|
||||
# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0)
|
||||
# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file.
|
||||
|
|
|
@ -27,6 +27,7 @@
|
|||
# SPARK_PID_DIR The pid files are stored. /tmp by default.
|
||||
# SPARK_IDENT_STRING A string representing this instance of spark. $USER by default
|
||||
# SPARK_NICENESS The scheduling priority for daemons. Defaults to 0.
|
||||
# SPARK_NO_DAEMONIZE If set, will run the proposed command in the foreground. It will not output a PID file.
|
||||
##
|
||||
|
||||
usage="Usage: spark-daemon.sh [--config <conf-dir>] (start|stop|submit|status) <spark-command> <spark-instance-number> <args...>"
|
||||
|
@ -122,6 +123,35 @@ if [ "$SPARK_NICENESS" = "" ]; then
|
|||
export SPARK_NICENESS=0
|
||||
fi
|
||||
|
||||
execute_command() {
|
||||
local command="$@"
|
||||
if [ -z ${SPARK_NO_DAEMONIZE+set} ]; then
|
||||
nohup -- $command >> $log 2>&1 < /dev/null &
|
||||
newpid="$!"
|
||||
|
||||
echo "$newpid" > "$pid"
|
||||
|
||||
# Poll for up to 5 seconds for the java process to start
|
||||
for i in {1..10}
|
||||
do
|
||||
if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
sleep 2
|
||||
# Check if the process has died; in that case we'll tail the log so the user can see
|
||||
if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
|
||||
echo "failed to launch $command:"
|
||||
tail -2 "$log" | sed 's/^/ /'
|
||||
echo "full log in $log"
|
||||
fi
|
||||
else
|
||||
$command
|
||||
fi
|
||||
}
|
||||
|
||||
run_command() {
|
||||
mode="$1"
|
||||
shift
|
||||
|
@ -146,13 +176,11 @@ run_command() {
|
|||
|
||||
case "$mode" in
|
||||
(class)
|
||||
nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null &
|
||||
newpid="$!"
|
||||
execute_command nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command $@
|
||||
;;
|
||||
|
||||
(submit)
|
||||
nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null &
|
||||
newpid="$!"
|
||||
execute_command nice -n "$SPARK_NICENESS" bash "${SPARK_HOME}"/bin/spark-submit --class $command $@
|
||||
;;
|
||||
|
||||
(*)
|
||||
|
@ -161,24 +189,6 @@ run_command() {
|
|||
;;
|
||||
esac
|
||||
|
||||
echo "$newpid" > "$pid"
|
||||
|
||||
#Poll for up to 5 seconds for the java process to start
|
||||
for i in {1..10}
|
||||
do
|
||||
if [[ $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
sleep 2
|
||||
# Check if the process has died; in that case we'll tail the log so the user can see
|
||||
if [[ ! $(ps -p "$newpid" -o comm=) =~ "java" ]]; then
|
||||
echo "failed to launch $command:"
|
||||
tail -2 "$log" | sed 's/^/ /'
|
||||
echo "full log in $log"
|
||||
fi
|
||||
}
|
||||
|
||||
case $option in
|
||||
|
|
Loading…
Reference in a new issue