ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-9870] Disable driver UI and Master REST server in SparkSubmitSuite

Browse source 
I think that we should pass additional configuration flags to disable the driver UI and Master REST server in SparkSubmitSuite and HiveSparkSubmitSuite. This might cut down on port-contention-related flakiness in Jenkins.

Author: Josh Rosen <joshrosen@databricks.com>

Closes #8124 from JoshRosen/disable-ui-in-sparksubmitsuite.
This commit is contained in:
Josh Rosen 2015-08-12 18:52:11 -07:00 committed by Andrew Or
parent f4bc01f1f3
commit 7b13ed27c1
2 changed files with 16 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
View file

@ -324,6 +324,8 @@ class SparkSubmitSuite
"--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
"--name", "testApp",
"--master", "local",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
unusedJar.toString)
runSparkSubmit(args)
}
@ -337,6 +339,8 @@ class SparkSubmitSuite
"--class", JarCreationTest.getClass.getName.stripSuffix("$"),
"--name", "testApp",
"--master", "local-cluster[2,1,1024]",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--jars", jarsString,
unusedJar.toString, "SparkSubmitClassA", "SparkSubmitClassB")
runSparkSubmit(args)
@ -355,6 +359,7 @@ class SparkSubmitSuite
"--packages", Seq(main, dep).mkString(","),
"--repositories", repo,
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
unusedJar.toString,
"my.great.lib.MyLib", "my.great.dep.MyLib")
runSparkSubmit(args)
@ -500,6 +505,8 @@ class SparkSubmitSuite
"--master", "local",
"--conf", "spark.driver.extraClassPath=" + systemJar,
"--conf", "spark.driver.userClassPathFirst=true",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
userJar.toString)
runSparkSubmit(args)
}

10
sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
View file

@ -57,6 +57,8 @@ class HiveSparkSubmitSuite
"--class", SparkSubmitClassLoaderTest.getClass.getName.stripSuffix("$"),
"--name", "SparkSubmitClassLoaderTest",
"--master", "local-cluster[2,1,1024]",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--jars", jarsString,
unusedJar.toString, "SparkSubmitClassA", "SparkSubmitClassB")
runSparkSubmit(args)
@ -68,6 +70,8 @@ class HiveSparkSubmitSuite
"--class", SparkSQLConfTest.getClass.getName.stripSuffix("$"),
"--name", "SparkSQLConfTest",
"--master", "local-cluster[2,1,1024]",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
unusedJar.toString)
runSparkSubmit(args)
}
@ -79,7 +83,11 @@ class HiveSparkSubmitSuite
// the HiveContext code mistakenly overrides the class loader that contains user classes.
// For more detail, see sql/hive/src/test/resources/regression-test-SPARK-8489/*scala.
val testJar = "sql/hive/src/test/resources/regression-test-SPARK-8489/test.jar"
val args = Seq("--class", "Main", testJar)
val args = Seq(
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
"--class", "Main",
testJar)
runSparkSubmit(args)
}