[SPARK-29436][K8S] Support executor for selecting scheduler through scheduler name in the case of k8s multi-scheduler scenario
### What changes were proposed in this pull request? Support executor for selecting scheduler through scheduler name in the case of k8s multi-scheduler scenario. ### Why are the changes needed? If there is no such function, spark can not support the case of k8s multi-scheduler scenario. ### Does this PR introduce any user-facing change? Yes, users can add scheduler name through configuration. ### How was this patch tested? Manually tested with spark + k8s cluster Closes #26088 from merrily01/SPARK-29436. Authored-by: maruilei <maruilei@jd.com> Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
parent
dc0bc7a6eb
commit
f800fa3831
|
@ -142,6 +142,12 @@ private[spark] object Config extends Logging {
|
||||||
.stringConf
|
.stringConf
|
||||||
.createOptional
|
.createOptional
|
||||||
|
|
||||||
|
val KUBERNETES_EXECUTOR_SCHEDULER_NAME =
|
||||||
|
ConfigBuilder("spark.kubernetes.executor.scheduler.name")
|
||||||
|
.doc("Specify the scheduler name for each executor pod")
|
||||||
|
.stringConf
|
||||||
|
.createOptional
|
||||||
|
|
||||||
val KUBERNETES_EXECUTOR_REQUEST_CORES =
|
val KUBERNETES_EXECUTOR_REQUEST_CORES =
|
||||||
ConfigBuilder("spark.kubernetes.executor.request.cores")
|
ConfigBuilder("spark.kubernetes.executor.request.cores")
|
||||||
.doc("Specify the cpu request for each executor pod")
|
.doc("Specify the cpu request for each executor pod")
|
||||||
|
|
|
@ -216,6 +216,9 @@ private[spark] class BasicExecutorFeatureStep(
|
||||||
.endSpec()
|
.endSpec()
|
||||||
.build()
|
.build()
|
||||||
|
|
||||||
|
kubernetesConf.get(KUBERNETES_EXECUTOR_SCHEDULER_NAME)
|
||||||
|
.foreach(executorPod.getSpec.setSchedulerName)
|
||||||
|
|
||||||
SparkPod(executorPod, containerWithLimitCores)
|
SparkPod(executorPod, containerWithLimitCores)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue