[SPARK-25088][CORE][MESOS][DOCS] Update Rest Server docs & defaults.
## What changes were proposed in this pull request? (a) disabled rest submission server by default in standalone mode (b) fails the standalone master if rest server enabled & authentication secret set (c) fails the mesos cluster dispatcher if authentication secret set (d) doc updates (e) when submitting a standalone app, only try the rest submission first if spark.master.rest.enabled=true otherwise you'd see a 10 second pause like 18/08/09 08:13:22 INFO RestSubmissionClient: Submitting a request to launch an application in spark://... 18/08/09 08:13:33 WARN RestSubmissionClient: Unable to connect to server spark://... I also made sure the mesos cluster dispatcher failed with the secret enabled, though I had to do that on slightly different code as I don't have mesos native libs around. ## How was this patch tested? I ran the tests in the mesos module & in core for org.apache.spark.deploy.* I ran a test on a cluster with standalone master to make sure I could still start with the right configs, and would fail the right way too. Closes #22071 from squito/rest_doc_updates. Authored-by: Imran Rashid <irashid@cloudera.com> Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
parent
80784a1de8
commit
1024875843
|
@ -82,7 +82,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
|
||||||
var driverCores: String = null
|
var driverCores: String = null
|
||||||
var submissionToKill: String = null
|
var submissionToKill: String = null
|
||||||
var submissionToRequestStatusFor: String = null
|
var submissionToRequestStatusFor: String = null
|
||||||
var useRest: Boolean = true // used internally
|
var useRest: Boolean = false // used internally
|
||||||
|
|
||||||
/** Default properties present in the currently defined defaults file. */
|
/** Default properties present in the currently defined defaults file. */
|
||||||
lazy val defaultSparkProperties: HashMap[String, String] = {
|
lazy val defaultSparkProperties: HashMap[String, String] = {
|
||||||
|
@ -115,6 +115,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
|
||||||
// Use `sparkProperties` map along with env vars to fill in any missing parameters
|
// Use `sparkProperties` map along with env vars to fill in any missing parameters
|
||||||
loadEnvironmentArguments()
|
loadEnvironmentArguments()
|
||||||
|
|
||||||
|
useRest = sparkProperties.getOrElse("spark.master.rest.enabled", "false").toBoolean
|
||||||
|
|
||||||
validateArguments()
|
validateArguments()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -121,10 +121,18 @@ private[deploy] class Master(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Alternative application submission gateway that is stable across Spark versions
|
// Alternative application submission gateway that is stable across Spark versions
|
||||||
private val restServerEnabled = conf.getBoolean("spark.master.rest.enabled", true)
|
private val restServerEnabled = conf.getBoolean("spark.master.rest.enabled", false)
|
||||||
private var restServer: Option[StandaloneRestServer] = None
|
private var restServer: Option[StandaloneRestServer] = None
|
||||||
private var restServerBoundPort: Option[Int] = None
|
private var restServerBoundPort: Option[Int] = None
|
||||||
|
|
||||||
|
{
|
||||||
|
val authKey = SecurityManager.SPARK_AUTH_SECRET_CONF
|
||||||
|
require(conf.getOption(authKey).isEmpty || !restServerEnabled,
|
||||||
|
s"The RestSubmissionServer does not support authentication via ${authKey}. Either turn " +
|
||||||
|
"off the RestSubmissionServer with spark.master.rest.enabled=false, or do not use " +
|
||||||
|
"authentication.")
|
||||||
|
}
|
||||||
|
|
||||||
override def onStart(): Unit = {
|
override def onStart(): Unit = {
|
||||||
logInfo("Starting Spark master at " + masterUrl)
|
logInfo("Starting Spark master at " + masterUrl)
|
||||||
logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
|
logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
|
||||||
|
|
|
@ -51,6 +51,7 @@ private[spark] abstract class RestSubmissionServer(
|
||||||
val host: String,
|
val host: String,
|
||||||
val requestedPort: Int,
|
val requestedPort: Int,
|
||||||
val masterConf: SparkConf) extends Logging {
|
val masterConf: SparkConf) extends Logging {
|
||||||
|
|
||||||
protected val submitRequestServlet: SubmitRequestServlet
|
protected val submitRequestServlet: SubmitRequestServlet
|
||||||
protected val killRequestServlet: KillRequestServlet
|
protected val killRequestServlet: KillRequestServlet
|
||||||
protected val statusRequestServlet: StatusRequestServlet
|
protected val statusRequestServlet: StatusRequestServlet
|
||||||
|
|
|
@ -174,6 +174,8 @@ can find the results of the driver from the Mesos Web UI.
|
||||||
|
|
||||||
To use cluster mode, you must start the `MesosClusterDispatcher` in your cluster via the `sbin/start-mesos-dispatcher.sh` script,
|
To use cluster mode, you must start the `MesosClusterDispatcher` in your cluster via the `sbin/start-mesos-dispatcher.sh` script,
|
||||||
passing in the Mesos master URL (e.g: mesos://host:5050). This starts the `MesosClusterDispatcher` as a daemon running on the host.
|
passing in the Mesos master URL (e.g: mesos://host:5050). This starts the `MesosClusterDispatcher` as a daemon running on the host.
|
||||||
|
Note that the `MesosClusterDispatcher` does not support authentication. You should ensure that all network access to it is
|
||||||
|
protected (port 7077 by default).
|
||||||
|
|
||||||
By setting the Mesos proxy config property (requires mesos version >= 1.4), `--conf spark.mesos.proxy.baseURL=http://localhost:5050` when launching the dispatcher, the mesos sandbox URI for each driver is added to the mesos dispatcher UI.
|
By setting the Mesos proxy config property (requires mesos version >= 1.4), `--conf spark.mesos.proxy.baseURL=http://localhost:5050` when launching the dispatcher, the mesos sandbox URI for each driver is added to the mesos dispatcher UI.
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,12 @@ secrets to be secure.
|
||||||
|
|
||||||
For other resource managers, `spark.authenticate.secret` must be configured on each of the nodes.
|
For other resource managers, `spark.authenticate.secret` must be configured on each of the nodes.
|
||||||
This secret will be shared by all the daemons and applications, so this deployment configuration is
|
This secret will be shared by all the daemons and applications, so this deployment configuration is
|
||||||
not as secure as the above, especially when considering multi-tenant clusters.
|
not as secure as the above, especially when considering multi-tenant clusters. In this
|
||||||
|
configuration, a user with the secret can effectively impersonate any other user.
|
||||||
|
|
||||||
|
The Rest Submission Server and the MesosClusterDispatcher do not support authentication. You should
|
||||||
|
ensure that all network access to the REST API & MesosClusterDispatcher (port 6066 and 7077
|
||||||
|
respectively by default) are restricted to hosts that are trusted to submit jobs.
|
||||||
|
|
||||||
<table class="table">
|
<table class="table">
|
||||||
<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
|
<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
|
||||||
|
|
|
@ -51,6 +51,14 @@ private[mesos] class MesosClusterDispatcher(
|
||||||
conf: SparkConf)
|
conf: SparkConf)
|
||||||
extends Logging {
|
extends Logging {
|
||||||
|
|
||||||
|
{
|
||||||
|
// This doesn't support authentication because the RestSubmissionServer doesn't support it.
|
||||||
|
val authKey = SecurityManager.SPARK_AUTH_SECRET_CONF
|
||||||
|
require(conf.getOption(authKey).isEmpty,
|
||||||
|
s"The MesosClusterDispatcher does not support authentication via ${authKey}. It is not " +
|
||||||
|
s"currently possible to run jobs in cluster mode with authentication on.")
|
||||||
|
}
|
||||||
|
|
||||||
private val publicAddress = Option(conf.getenv("SPARK_PUBLIC_DNS")).getOrElse(args.host)
|
private val publicAddress = Option(conf.getenv("SPARK_PUBLIC_DNS")).getOrElse(args.host)
|
||||||
private val recoveryMode = conf.get(RECOVERY_MODE).toUpperCase()
|
private val recoveryMode = conf.get(RECOVERY_MODE).toUpperCase()
|
||||||
logInfo("Recovery mode in Mesos dispatcher set to: " + recoveryMode)
|
logInfo("Recovery mode in Mesos dispatcher set to: " + recoveryMode)
|
||||||
|
|
Loading…
Reference in a new issue