ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-26082][MESOS][FOLLOWUP] Add UT on fetcher cache option on MesosClusterScheduler

Browse source 
## What changes were proposed in this pull request?

This patch adds UT on testing SPARK-26082 to avoid regression. While #23743 reduces the possibility to make a similar mistake, the needed lines of code for adding tests are not that huge, so I guess it might be worth to add them.

## How was this patch tested?

Newly added UTs. Test "supports setting fetcher cache" fails when #23743 is not applied and succeeds when #23743 is applied.

Closes #23744 from HeartSaVioR/SPARK-26082-add-unit-test.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
Jungtaek Lim (HeartSaVioR) 2019-02-07 08:51:55 -08:00 committed by Dongjoon Hyun
parent c0811e8b4d
commit b4e1d14513
1 changed files with 48 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
resource-managers/mesos/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterSchedulerSuite.scala
View file

@ -255,6 +255,54 @@ class MesosClusterSchedulerSuite extends SparkFunSuite with LocalSparkContext wi
assert(networkInfos.get(0).getLabels.getLabels(1).getValue == "val2")
}
test("supports setting fetcher cache") {
setScheduler(Map("spark.mesos.fetcherCache.enable" -> "true"))
val mem = 1000
val cpu = 1
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", mem, cpu, true,
command,
Map("spark.mesos.executor.home" -> "test",
"spark.app.name" -> "test"),
"s1",
new Date()))
assert(response.success)
val offer = Utils.createOffer("o1", "s1", mem, cpu)
scheduler.resourceOffers(driver, List(offer).asJava)
val launchedTasks = Utils.verifyTaskLaunched(driver, "o1")
val uris = launchedTasks.head.getCommand.getUrisList
assert(uris.stream().allMatch(_.getCache))
}
test("supports disabling fetcher cache") {
setScheduler(Map("spark.mesos.fetcherCache.enable" -> "false"))
val mem = 1000
val cpu = 1
val response = scheduler.submitDriver(
new MesosDriverDescription("d1", "jar", mem, cpu, true,
command,
Map("spark.mesos.executor.home" -> "test",
"spark.app.name" -> "test"),
"s1",
new Date()))
assert(response.success)
val offer = Utils.createOffer("o1", "s1", mem, cpu)
scheduler.resourceOffers(driver, List(offer).asJava)
val launchedTasks = Utils.verifyTaskLaunched(driver, "o1")
val uris = launchedTasks.head.getCommand.getUrisList
assert(uris.stream().allMatch(!_.getCache))
}
test("accept/decline offers with driver constraints") {
setScheduler()