e34f38ff1a
The progress bar will look like this:  In the right corner, the numbers are: finished tasks, running tasks, total tasks. After the stage has finished, it will disappear. The progress bar is only showed if logging level is WARN or higher (but progress in title is still showed), it can be turned off by spark.driver.showConsoleProgress. Author: Davies Liu <davies@databricks.com> Closes #3029 from davies/progress and squashes the following commits: 95336d5 [Davies Liu] Merge branch 'master' of github.com:apache/spark into progress fc49ac8 [Davies Liu] address commentse 2e90f75 [Davies Liu] show multiple stages in same time 0081bcc [Davies Liu] address comments 38c42f1 [Davies Liu] fix tests ab87958 [Davies Liu] disable progress bar during tests 30ac852 [Davies Liu] re-implement progress bar b3f34e5 [Davies Liu] Merge branch 'master' of github.com:apache/spark into progress 6fd30ff [Davies Liu] show progress bar if no task finished in 500ms e4e7344 [Davies Liu] refactor e1f524d [Davies Liu] revert unnecessary change a60477c [Davies Liu] Merge branch 'master' of github.com:apache/spark into progress 5cae3f2 [Davies Liu] fix style ea49fe0 [Davies Liu] address comments bc53d99 [Davies Liu] refactor e6bb189 [Davies Liu] fix logging in sparkshell 7e7d4e7 [Davies Liu] address commments 5df26bb [Davies Liu] fix style 9e42208 [Davies Liu] show progress bar in console and title
67 lines
2.5 KiB
Bash
Executable file
67 lines
2.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# NOTE: Any changes in this file must be reflected in SparkSubmitDriverBootstrapper.scala!
|
|
|
|
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
|
|
ORIG_ARGS=("$@")
|
|
|
|
# Set COLUMNS for progress bar
|
|
export COLUMNS=`tput cols`
|
|
|
|
while (($#)); do
|
|
if [ "$1" = "--deploy-mode" ]; then
|
|
SPARK_SUBMIT_DEPLOY_MODE=$2
|
|
elif [ "$1" = "--properties-file" ]; then
|
|
SPARK_SUBMIT_PROPERTIES_FILE=$2
|
|
elif [ "$1" = "--driver-memory" ]; then
|
|
export SPARK_SUBMIT_DRIVER_MEMORY=$2
|
|
elif [ "$1" = "--driver-library-path" ]; then
|
|
export SPARK_SUBMIT_LIBRARY_PATH=$2
|
|
elif [ "$1" = "--driver-class-path" ]; then
|
|
export SPARK_SUBMIT_CLASSPATH=$2
|
|
elif [ "$1" = "--driver-java-options" ]; then
|
|
export SPARK_SUBMIT_OPTS=$2
|
|
fi
|
|
shift
|
|
done
|
|
|
|
DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf"
|
|
export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"}
|
|
export SPARK_SUBMIT_PROPERTIES_FILE=${SPARK_SUBMIT_PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"}
|
|
|
|
# For client mode, the driver will be launched in the same JVM that launches
|
|
# SparkSubmit, so we may need to read the properties file for any extra class
|
|
# paths, library paths, java options and memory early on. Otherwise, it will
|
|
# be too late by the time the driver JVM has started.
|
|
|
|
if [[ "$SPARK_SUBMIT_DEPLOY_MODE" == "client" && -f "$SPARK_SUBMIT_PROPERTIES_FILE" ]]; then
|
|
# Parse the properties file only if the special configs exist
|
|
contains_special_configs=$(
|
|
grep -e "spark.driver.extra*\|spark.driver.memory" "$SPARK_SUBMIT_PROPERTIES_FILE" | \
|
|
grep -v "^[[:space:]]*#"
|
|
)
|
|
if [ -n "$contains_special_configs" ]; then
|
|
export SPARK_SUBMIT_BOOTSTRAP_DRIVER=1
|
|
fi
|
|
fi
|
|
|
|
exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "${ORIG_ARGS[@]}"
|
|
|