ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-9672] [MESOS] Don’t include SPARK_ENV_LOADED when passing env vars

Browse source 
This contribution is my original work and I license the work to the project under the project's open source license.

Author: Pat Shields <yeoldefortran@gmail.com>

Closes #7979 from pashields/env-loading-on-driver.
This commit is contained in:
Pat Shields 2015-09-03 13:52:47 -07:00 committed by Andrew Or
parent 754f853b02
commit e62f4a46f4
2 changed files with 25 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
View file

@ -392,15 +392,14 @@ private[spark] object RestSubmissionClient {
mainClass: String,
appArgs: Array[String],
conf: SparkConf,
env: Map[String, String] = sys.env): SubmitRestProtocolResponse = {
env: Map[String, String] = Map()): SubmitRestProtocolResponse = {
val master = conf.getOption("spark.master").getOrElse {
throw new IllegalArgumentException("'spark.master' must be set.")
}
val sparkProperties = conf.getAll.toMap
val environmentVariables = env.filter { case (k, _) => k.startsWith("SPARK_") }
val client = new RestSubmissionClient(master)
val submitRequest = client.constructSubmitRequest(
appResource, mainClass, appArgs, sparkProperties, environmentVariables)
appResource, mainClass, appArgs, sparkProperties, env)
client.createSubmission(submitRequest)
}
@ -413,6 +412,16 @@ private[spark] object RestSubmissionClient {
val mainClass = args(1)
val appArgs = args.slice(2, args.size)
val conf = new SparkConf
run(appResource, mainClass, appArgs, conf)
val env = filterSystemEnvironment(sys.env)
run(appResource, mainClass, appArgs, conf, env)
}
/**
* Filter non-spark environment variables from any environment.
*/
private[rest] def filterSystemEnvironment(env: Map[String, String]): Map[String, String] = {
env.filter { case (k, _) =>
(k.startsWith("SPARK_") && k != "SPARK_ENV_LOADED") || k.startsWith("MESOS_")
}
}
}

12
core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
View file

@ -366,6 +366,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite with BeforeAndAfterEach {
assert(conn3.getResponseCode === HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
}
test("client does not send 'SPARK_ENV_LOADED' env var by default") {
val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_ENV_LOADED" -> "1")
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
assert(filteredVariables == Map("SPARK_VAR" -> "1"))
}
test("client includes mesos env vars") {
val environmentVariables = Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1", "OTHER_VAR" -> "1")
val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
assert(filteredVariables == Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1"))
}
/* --------------------- *
| Helper methods |
* --------------------- */