[SPARK-3658][SQL] Start thrift server as a daemon
https://issues.apache.org/jira/browse/SPARK-3658 And keep the `CLASS_NOT_FOUND_EXIT_STATUS` and exit message in `SparkSubmit.scala`. Author: WangTaoTheTonic <barneystinson@aliyun.com> Author: WangTao <barneystinson@aliyun.com> Closes #2509 from WangTaoTheTonic/thriftserver and squashes the following commits: 5dcaab2 [WangTaoTheTonic] issue about coupling 8ad9f95 [WangTaoTheTonic] generalization 598e21e [WangTao] take thrift server as a daemon
This commit is contained in:
WangTaoTheTonic
Michael Armbrust
parent
fcad3fae61
commit
d61f2c15bb
|
|
@ -24,7 +24,6 @@
|
|||
set -o posix
|
||||
|
||||
CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver"
|
||||
CLASS_NOT_FOUND_EXIT_STATUS=101
|
||||
|
||||
# Figure out where Spark is installed
|
||||
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
|
||||
|
|
@ -53,13 +52,4 @@ source "$FWDIR"/bin/utils.sh
|
|||
SUBMIT_USAGE_FUNCTION=usage
|
||||
gatherSparkSubmitOpts "$@"
|
||||
|
||||
"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}"
|
||||
exit_status=$?
|
||||
|
||||
if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then
|
||||
echo
|
||||
echo "Failed to load Spark SQL CLI main class $CLASS."
|
||||
echo "You need to build Spark with -Phive."
|
||||
fi
|
||||
|
||||
exit $exit_status
|
||||
exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}"
|
||||
|
|
|
|||
|
|
@ -320,6 +320,10 @@ object SparkSubmit {
|
|||
} catch {
|
||||
case e: ClassNotFoundException =>
|
||||
e.printStackTrace(printStream)
|
||||
if (childMainClass.contains("thriftserver")) {
|
||||
println(s"Failed to load main class $childMainClass.")
|
||||
println("You need to build Spark with -Phive.")
|
||||
}
|
||||
System.exit(CLASS_NOT_FOUND_EXIT_STATUS)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ then
|
|||
shift
|
||||
fi
|
||||
|
||||
startStop=$1
|
||||
option=$1
|
||||
shift
|
||||
command=$1
|
||||
shift
|
||||
|
|
@ -122,9 +122,9 @@ if [ "$SPARK_NICENESS" = "" ]; then
|
|||
fi
|
||||
|
||||
|
||||
case $startStop in
|
||||
case $option in
|
||||
|
||||
(start)
|
||||
(start|spark-submit)
|
||||
|
||||
mkdir -p "$SPARK_PID_DIR"
|
||||
|
||||
|
|
@ -142,8 +142,14 @@ case $startStop in
|
|||
|
||||
spark_rotate_log "$log"
|
||||
echo starting $command, logging to $log
|
||||
cd "$SPARK_PREFIX"
|
||||
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null &
|
||||
if [ $option == spark-submit ]; then
|
||||
source "$SPARK_HOME"/bin/utils.sh
|
||||
gatherSparkSubmitOpts "$@"
|
||||
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \
|
||||
"${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null &
|
||||
else
|
||||
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null &
|
||||
fi
|
||||
newpid=$!
|
||||
echo $newpid > $pid
|
||||
sleep 2
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ set -o posix
|
|||
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
|
||||
|
||||
CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"
|
||||
CLASS_NOT_FOUND_EXIT_STATUS=101
|
||||
|
||||
function usage {
|
||||
echo "Usage: ./sbin/start-thriftserver [options] [thrift server options]"
|
||||
|
|
@ -49,17 +48,6 @@ if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
source "$FWDIR"/bin/utils.sh
|
||||
SUBMIT_USAGE_FUNCTION=usage
|
||||
gatherSparkSubmitOpts "$@"
|
||||
export SUBMIT_USAGE_FUNCTION=usage
|
||||
|
||||
"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}"
|
||||
exit_status=$?
|
||||
|
||||
if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then
|
||||
echo
|
||||
echo "Failed to load Hive Thrift server main class $CLASS."
|
||||
echo "You need to build Spark with -Phive."
|
||||
fi
|
||||
|
||||
exit $exit_status
|
||||
exec "$FWDIR"/sbin/spark-daemon.sh spark-submit $CLASS 1 "$@"
|
||||
|
|
|
|||
25
sbin/stop-thriftserver.sh
Executable file
25
sbin/stop-thriftserver.sh
Executable file
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Stops the thrift server on the machine this script is executed on.
|
||||
|
||||
sbin="`dirname "$0"`"
|
||||
sbin="`cd "$sbin"; pwd`"
|
||||
|
||||
"$sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1
|
||||
Loading…
Reference in a new issue