diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index cec8272bee..7f34f30d59 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -96,7 +96,7 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf) val driverPort = conf.sparkConf.getInt(DRIVER_PORT.key, DEFAULT_DRIVER_PORT) val driverBlockManagerPort = conf.sparkConf.getInt( DRIVER_BLOCK_MANAGER_PORT.key, - DEFAULT_BLOCKMANAGER_PORT + conf.sparkConf.getInt(BLOCK_MANAGER_PORT.key, DEFAULT_BLOCKMANAGER_PORT) ) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) val driverContainer = new ContainerBuilder(pod.container) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala index 858b4f1494..f0843225ea 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala @@ -213,6 +213,25 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { } } + test("SPARK-35493: make spark.blockManager.port be able to be fallen back to in driver pod") { + val initPod = SparkPod.initialPod() + val sparkConf = new SparkConf() + .set(CONTAINER_IMAGE, "spark-driver:latest") + .set(BLOCK_MANAGER_PORT, 1234) + val driverConf1 = KubernetesTestConf.createDriverConf(sparkConf) + val pod1 = new BasicDriverFeatureStep(driverConf1).configurePod(initPod) + val portMap1 = + pod1.container.getPorts.asScala.map { cp => (cp.getName -> cp.getContainerPort) }.toMap + assert(portMap1(BLOCK_MANAGER_PORT_NAME) === 1234, s"fallback to $BLOCK_MANAGER_PORT.key") + + val driverConf2 = + KubernetesTestConf.createDriverConf(sparkConf.set(DRIVER_BLOCK_MANAGER_PORT, 1235)) + val pod2 = new BasicDriverFeatureStep(driverConf2).configurePod(initPod) + val portMap2 = + pod2.container.getPorts.asScala.map { cp => (cp.getName -> cp.getContainerPort) }.toMap + assert(portMap2(BLOCK_MANAGER_PORT_NAME) === 1235) + } + def containerPort(name: String, portNumber: Int): ContainerPort = new ContainerPortBuilder() .withName(name)