2013-07-17 20:51:18 -04:00
|
|
|
#!/usr/bin/env bash
|
2013-07-16 20:21:33 -04:00
|
|
|
|
|
|
|
#
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
|
|
# this work for additional information regarding copyright ownership.
|
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
# (the "License"); you may not use this file except in compliance with
|
|
|
|
# the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
#
|
|
|
|
|
2013-06-25 14:14:21 -04:00
|
|
|
#
|
|
|
|
# Shell script for starting the Spark Shell REPL
|
2013-12-15 21:39:30 -05:00
|
|
|
|
|
|
|
cygwin=false
|
2016-11-16 17:22:15 -05:00
|
|
|
case "$(uname)" in
|
2014-08-24 12:43:44 -04:00
|
|
|
CYGWIN*) cygwin=true;;
|
2013-12-15 21:39:30 -05:00
|
|
|
esac
|
2013-07-16 20:21:33 -04:00
|
|
|
|
2013-07-17 20:51:18 -04:00
|
|
|
# Enter posix mode for bash
|
|
|
|
set -o posix
|
|
|
|
|
2015-11-04 05:49:34 -05:00
|
|
|
if [ -z "${SPARK_HOME}" ]; then
|
2016-11-16 17:22:15 -05:00
|
|
|
source "$(dirname "$0")"/find-spark-home
|
2015-11-04 05:49:34 -05:00
|
|
|
fi
|
|
|
|
|
[SPARK-25906][SHELL] Documents '-I' option (from Scala REPL) in spark-shell
## What changes were proposed in this pull request?
This PR targets to document `-I` option from Spark 2.4.x (previously `-i` option until Spark 2.3.x).
After we upgraded Scala to 2.11.12, `-i` option (`:load`) was replaced to `-I`(SI-7898). Existing `-i` became `:paste` which does not respect Spark's implicit import (for instance `toDF`, symbol as column, etc.). Therefore, `-i` option does not correctly from Spark 2.4.x and it's not documented.
I checked other Scala REPL options but looks not applicable or working from quick tests. This PR only targets to document `-I` for now.
## How was this patch tested?
Manually tested.
**Mac:**
```bash
$ ./bin/spark-shell --help
Usage: ./bin/spark-shell [options]
Scala REPL options:
-I <file> preload <file>, enforcing line-by-line interpretation
Options:
--master MASTER_URL spark://host:port, mesos://host:port, yarn,
k8s://https://host:port, or local (Default: local[*]).
--deploy-mode DEPLOY_MODE Whether to launch the driver program locally ("client") or
on one of the worker machines inside the cluster ("cluster")
(Default: client).
...
```
**Windows:**
```cmd
C:\...\spark>.\bin\spark-shell --help
Usage: .\bin\spark-shell.cmd [options]
Scala REPL options:
-I <file> preload <file>, enforcing line-by-line interpretation
Options:
--master MASTER_URL spark://host:port, mesos://host:port, yarn,
k8s://https://host:port, or local (Default: local[*]).
--deploy-mode DEPLOY_MODE Whether to launch the driver program locally ("client") or
on one of the worker machines inside the cluster ("cluster")
(Default: client).
...
```
Closes #22919 from HyukjinKwon/SPARK-25906.
Authored-by: hyukjinkwon <gurwls223@apache.org>
Signed-off-by: hyukjinkwon <gurwls223@apache.org>
2018-11-05 21:39:58 -05:00
|
|
|
export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]
|
|
|
|
|
|
|
|
Scala REPL options:
|
|
|
|
-I <file> preload <file>, enforcing line-by-line interpretation"
|
2014-04-25 02:59:16 -04:00
|
|
|
|
2014-12-10 15:24:04 -05:00
|
|
|
# SPARK-4161: scala does not assume use of the java classpath,
|
2015-03-11 04:03:01 -04:00
|
|
|
# so we need to add the "-Dscala.usejavacp=true" flag manually. We
|
2014-12-10 15:24:04 -05:00
|
|
|
# do this specifically for the Spark shell because the scala REPL
|
|
|
|
# has its own class loader, and any additional classpath specified
|
|
|
|
# through spark.driver.extraClassPath is not automatically propagated.
|
|
|
|
SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dscala.usejavacp=true"
|
|
|
|
|
2014-08-10 00:10:43 -04:00
|
|
|
function main() {
|
2014-08-24 12:43:44 -04:00
|
|
|
if $cygwin; then
|
|
|
|
# Workaround for issue involving JLine and Cygwin
|
|
|
|
# (see http://sourceforge.net/p/jline/bugs/40/).
|
|
|
|
# If you're using the Mintty terminal emulator in Cygwin, may need to set the
|
|
|
|
# "Backspace sends ^H" setting in "Keys" section of the Mintty options
|
|
|
|
# (see https://github.com/sbt/sbt/issues/562).
|
|
|
|
stty -icanon min 1 -echo > /dev/null 2>&1
|
|
|
|
export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix"
|
2015-11-04 05:49:34 -05:00
|
|
|
"${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
|
2014-08-24 12:43:44 -04:00
|
|
|
stty icanon echo > /dev/null 2>&1
|
|
|
|
else
|
|
|
|
export SPARK_SUBMIT_OPTS
|
2015-11-04 05:49:34 -05:00
|
|
|
"${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
|
2014-08-24 12:43:44 -04:00
|
|
|
fi
|
2014-03-29 22:49:22 -04:00
|
|
|
}
|
|
|
|
|
2013-06-25 02:55:09 -04:00
|
|
|
# Copy restore-TTY-on-exit functions from Scala script so spark-shell exits properly even in
|
|
|
|
# binary distribution of Spark where Scala is not installed
|
|
|
|
exit_status=127
|
|
|
|
saved_stty=""
|
|
|
|
|
|
|
|
# restore stty settings (echo in particular)
|
|
|
|
function restoreSttySettings() {
|
|
|
|
stty $saved_stty
|
|
|
|
saved_stty=""
|
|
|
|
}
|
|
|
|
|
|
|
|
function onExit() {
|
|
|
|
if [[ "$saved_stty" != "" ]]; then
|
|
|
|
restoreSttySettings
|
|
|
|
fi
|
|
|
|
exit $exit_status
|
|
|
|
}
|
|
|
|
|
|
|
|
# to reenable echo if we are interrupted before completing.
|
|
|
|
trap onExit INT
|
|
|
|
|
|
|
|
# save terminal settings
|
|
|
|
saved_stty=$(stty -g 2>/dev/null)
|
|
|
|
# clear on error so we don't later try to restore them
|
|
|
|
if [[ ! $? ]]; then
|
|
|
|
saved_stty=""
|
|
|
|
fi
|
|
|
|
|
2014-04-28 20:29:22 -04:00
|
|
|
main "$@"
|
2013-06-25 02:55:09 -04:00
|
|
|
|
|
|
|
# record the exit status lest it be overwritten:
|
|
|
|
# then reenable echo and propagate the code.
|
|
|
|
exit_status=$?
|
|
|
|
onExit
|
2014-03-29 22:49:22 -04:00
|
|
|
|