[SPARK-20078][MESOS] Mesos executor configurability for task name and labels
## What changes were proposed in this pull request? Adding configurable mesos executor names and labels using `spark.mesos.task.name` and `spark.mesos.task.labels`. Labels were defined as `k1:v1,k2:v2`. mgummelt ## How was this patch tested? Added unit tests to verify labels were added correctly, with incorrect labels being ignored and added a test to test the name of the executor. Tested with: `./build/sbt -Pmesos mesos/test` Please review http://spark.apache.org/contributing.html before opening a pull request. Author: Kalvin Chau <kalvin.chau@viasat.com> Closes #17404 from kalvinnchau/mesos-config.
This commit is contained in:
parent
a2ce0a2e30
commit
e8ddb91c7e
|
@ -403,7 +403,8 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
|
||||||
.setTaskId(TaskID.newBuilder().setValue(taskId.toString).build())
|
.setTaskId(TaskID.newBuilder().setValue(taskId.toString).build())
|
||||||
.setSlaveId(offer.getSlaveId)
|
.setSlaveId(offer.getSlaveId)
|
||||||
.setCommand(createCommand(offer, taskCPUs + extraCoresPerExecutor, taskId))
|
.setCommand(createCommand(offer, taskCPUs + extraCoresPerExecutor, taskId))
|
||||||
.setName("Task " + taskId)
|
.setName(s"${sc.appName} $taskId")
|
||||||
|
|
||||||
taskBuilder.addAllResources(resourcesToUse.asJava)
|
taskBuilder.addAllResources(resourcesToUse.asJava)
|
||||||
taskBuilder.setContainer(MesosSchedulerBackendUtil.containerInfo(sc.conf))
|
taskBuilder.setContainer(MesosSchedulerBackendUtil.containerInfo(sc.conf))
|
||||||
|
|
||||||
|
|
|
@ -464,6 +464,17 @@ class MesosCoarseGrainedSchedulerBackendSuite extends SparkFunSuite
|
||||||
assert(!uris.asScala.head.getCache)
|
assert(!uris.asScala.head.getCache)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test("mesos sets task name to spark.app.name") {
|
||||||
|
setBackend()
|
||||||
|
|
||||||
|
val offers = List(Resources(backend.executorMemory(sc), 1))
|
||||||
|
offerResources(offers)
|
||||||
|
val launchedTasks = verifyTaskLaunched(driver, "o1")
|
||||||
|
|
||||||
|
// Add " 0" to the taskName to match the executor number that is appended
|
||||||
|
assert(launchedTasks.head.getName == "test-mesos-dynamic-alloc 0")
|
||||||
|
}
|
||||||
|
|
||||||
test("mesos supports spark.mesos.network.name") {
|
test("mesos supports spark.mesos.network.name") {
|
||||||
setBackend(Map(
|
setBackend(Map(
|
||||||
"spark.mesos.network.name" -> "test-network-name"
|
"spark.mesos.network.name" -> "test-network-name"
|
||||||
|
|
Loading…
Reference in a new issue