[SPARK-7037] [CORE] Inconsistent behavior for non-spark config properties in spark-shell and spark-submit
When specifying non-spark properties (i.e. names don't start with spark.) in the command line and config file, spark-submit and spark-shell behave differently, causing confusion to users. Here is the summary- * spark-submit * --conf k=v => silently ignored * spark-defaults.conf => applied * spark-shell * --conf k=v => show a warning message and ignored * spark-defaults.conf => show a warning message and ignored I assume that ignoring non-spark properties is intentional. If so, it should always be ignored with a warning message in all cases. Author: Cheolsoo Park <cheolsoop@netflix.com> Closes #5617 from piaozhexiu/SPARK-7037 and squashes the following commits: 8957950 [Cheolsoo Park] Add IgnoreNonSparkProperties method fedd01c [Cheolsoo Park] Ignore non-spark properties with a warning message in all cases
This commit is contained in:
parent
73db132bf5
commit
336f7f5373
|
@ -77,12 +77,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
|
|||
if (verbose) SparkSubmit.printStream.println(s"Using properties file: $propertiesFile")
|
||||
Option(propertiesFile).foreach { filename =>
|
||||
Utils.getPropertiesFromFile(filename).foreach { case (k, v) =>
|
||||
if (k.startsWith("spark.")) {
|
||||
defaultProperties(k) = v
|
||||
if (verbose) SparkSubmit.printStream.println(s"Adding default property: $k=$v")
|
||||
} else {
|
||||
SparkSubmit.printWarning(s"Ignoring non-spark config property: $k=$v")
|
||||
}
|
||||
}
|
||||
}
|
||||
defaultProperties
|
||||
|
@ -97,6 +93,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
|
|||
}
|
||||
// Populate `sparkProperties` map from properties file
|
||||
mergeDefaultSparkProperties()
|
||||
// Remove keys that don't start with "spark." from `sparkProperties`.
|
||||
ignoreNonSparkProperties()
|
||||
// Use `sparkProperties` map along with env vars to fill in any missing parameters
|
||||
loadEnvironmentArguments()
|
||||
|
||||
|
@ -117,6 +115,18 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove keys that don't start with "spark." from `sparkProperties`.
|
||||
*/
|
||||
private def ignoreNonSparkProperties(): Unit = {
|
||||
sparkProperties.foreach { case (k, v) =>
|
||||
if (!k.startsWith("spark.")) {
|
||||
sparkProperties -= k
|
||||
SparkSubmit.printWarning(s"Ignoring non-spark config property: $k=$v")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load arguments from environment variables, Spark properties etc.
|
||||
*/
|
||||
|
|
Loading…
Reference in a new issue