[SPARK-25934][MESOS] Don't propagate SPARK_CONF_DIR from spark submit
## What changes were proposed in this pull request? Don't propagate SPARK_CONF_DIR to the driver in mesos cluster mode. ## How was this patch tested? I built the 2.3.2 tag with this patch added and deployed a test job to a mesos cluster to confirm that the incorrect SPARK_CONF_DIR was no longer passed from the submit command. Closes #22937 from mpmolek/fix-conf-dir. Authored-by: Matt Molek <mpmolek@gmail.com> Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
parent
2aef79a65a
commit
696b75a810
|
@ -408,6 +408,10 @@ private[spark] class RestSubmissionClient(master: String) extends Logging {
|
|||
}
|
||||
|
||||
private[spark] object RestSubmissionClient {
|
||||
|
||||
// SPARK_HOME and SPARK_CONF_DIR are filtered out because they are usually wrong
|
||||
// on the remote machine (SPARK-12345) (SPARK-25934)
|
||||
private val BLACKLISTED_SPARK_ENV_VARS = Set("SPARK_ENV_LOADED", "SPARK_HOME", "SPARK_CONF_DIR")
|
||||
private val REPORT_DRIVER_STATUS_INTERVAL = 1000
|
||||
private val REPORT_DRIVER_STATUS_MAX_TRIES = 10
|
||||
val PROTOCOL_VERSION = "v1"
|
||||
|
@ -417,9 +421,7 @@ private[spark] object RestSubmissionClient {
|
|||
*/
|
||||
private[rest] def filterSystemEnvironment(env: Map[String, String]): Map[String, String] = {
|
||||
env.filterKeys { k =>
|
||||
// SPARK_HOME is filtered out because it is usually wrong on the remote machine (SPARK-12345)
|
||||
(k.startsWith("SPARK_") && k != "SPARK_ENV_LOADED" && k != "SPARK_HOME") ||
|
||||
k.startsWith("MESOS_")
|
||||
(k.startsWith("SPARK_") && !BLACKLISTED_SPARK_ENV_VARS.contains(k)) || k.startsWith("MESOS_")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -396,6 +396,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite with BeforeAndAfterEach {
|
|||
assert(filteredVariables == Map("SPARK_VAR" -> "1"))
|
||||
}
|
||||
|
||||
test("client does not send 'SPARK_HOME' env var by default") {
|
||||
val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_HOME" -> "1")
|
||||
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
|
||||
assert(filteredVariables == Map("SPARK_VAR" -> "1"))
|
||||
}
|
||||
|
||||
test("client does not send 'SPARK_CONF_DIR' env var by default") {
|
||||
val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_CONF_DIR" -> "1")
|
||||
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
|
||||
assert(filteredVariables == Map("SPARK_VAR" -> "1"))
|
||||
}
|
||||
|
||||
test("client includes mesos env vars") {
|
||||
val environmentVariables = Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1", "OTHER_VAR" -> "1")
|
||||
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
|
||||
|
|
Loading…
Reference in a new issue