ed167e70c6
Hadoop has a feature that allows users to impersonate other users
when submitting applications or talking to HDFS, for example. These
impersonated users are referred generally as "proxy users".
Services such as Oozie or Hive use this feature to run applications
as the requesting user.
This change makes SparkSubmit accept a new command line option to
run the application as a proxy user. It also fixes the plumbing
of the user name through the UI (and a couple of other places) to
refer to the correct user running the application, which can be
different than `sys.props("user.name")` even without proxies (e.g.
when using kerberos).
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #4405 from vanzin/SPARK-5493 and squashes the following commits:
df82427 [Marcelo Vanzin] Clarify the reason for the special exception handling.
05bfc08 [Marcelo Vanzin] Remove unneeded annotation.
4840de9 [Marcelo Vanzin] Review feedback.
8af06ff [Marcelo Vanzin] Fix usage string.
2e4fa8f [Marcelo Vanzin] Merge branch 'master' into SPARK-5493
b6c947d [Marcelo Vanzin] Merge branch 'master' into SPARK-5493
0540d38 [Marcelo Vanzin] [SPARK-5493] [core] Add option to impersonate user.
61 lines
2.1 KiB
Bash
Executable file
61 lines
2.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Gather all spark-submit options into SUBMISSION_OPTS
|
|
function gatherSparkSubmitOpts() {
|
|
|
|
if [ -z "$SUBMIT_USAGE_FUNCTION" ]; then
|
|
echo "Function for printing usage of $0 is not set." 1>&2
|
|
echo "Please set usage function to shell variable 'SUBMIT_USAGE_FUNCTION' in $0" 1>&2
|
|
exit 1
|
|
fi
|
|
|
|
# NOTE: If you add or remove spark-submit options,
|
|
# modify NOT ONLY this script but also SparkSubmitArgument.scala
|
|
SUBMISSION_OPTS=()
|
|
APPLICATION_OPTS=()
|
|
while (($#)); do
|
|
case "$1" in
|
|
--master | --deploy-mode | --class | --name | --jars | --packages | --py-files | --files | \
|
|
--conf | --repositories | --properties-file | --driver-memory | --driver-java-options | \
|
|
--driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
|
|
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives | \
|
|
--proxy-user)
|
|
if [[ $# -lt 2 ]]; then
|
|
"$SUBMIT_USAGE_FUNCTION"
|
|
exit 1;
|
|
fi
|
|
SUBMISSION_OPTS+=("$1"); shift
|
|
SUBMISSION_OPTS+=("$1"); shift
|
|
;;
|
|
|
|
--verbose | -v | --supervise)
|
|
SUBMISSION_OPTS+=("$1"); shift
|
|
;;
|
|
|
|
*)
|
|
APPLICATION_OPTS+=("$1"); shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
export SUBMISSION_OPTS
|
|
export APPLICATION_OPTS
|
|
}
|