[SPARK-25828][K8S] Bumping Kubernetes-Client version to 4.1.0
## What changes were proposed in this pull request? Changed the `kubernetes-client` version and refactored code that broke as a result ## How was this patch tested? Unit and Integration tests Closes #22820 from ifilonenko/SPARK-25828. Authored-by: Ilan Filonenko <ifilondz@gmail.com> Signed-off-by: Erik Erlandson <eerlands@redhat.com>
This commit is contained in:
parent
ca545f7941
commit
e9b71c8f01
|
@ -132,13 +132,13 @@ jta-1.1.jar
|
||||||
jtransforms-2.4.0.jar
|
jtransforms-2.4.0.jar
|
||||||
jul-to-slf4j-1.7.16.jar
|
jul-to-slf4j-1.7.16.jar
|
||||||
kryo-shaded-4.0.2.jar
|
kryo-shaded-4.0.2.jar
|
||||||
kubernetes-client-3.0.0.jar
|
kubernetes-client-4.1.0.jar
|
||||||
kubernetes-model-2.0.0.jar
|
kubernetes-model-4.1.0.jar
|
||||||
leveldbjni-all-1.8.jar
|
leveldbjni-all-1.8.jar
|
||||||
libfb303-0.9.3.jar
|
libfb303-0.9.3.jar
|
||||||
libthrift-0.9.3.jar
|
libthrift-0.9.3.jar
|
||||||
log4j-1.2.17.jar
|
log4j-1.2.17.jar
|
||||||
logging-interceptor-3.8.1.jar
|
logging-interceptor-3.9.1.jar
|
||||||
lz4-java-1.5.0.jar
|
lz4-java-1.5.0.jar
|
||||||
machinist_2.11-0.6.1.jar
|
machinist_2.11-0.6.1.jar
|
||||||
macro-compat_2.11-1.1.1.jar
|
macro-compat_2.11-1.1.1.jar
|
||||||
|
|
|
@ -147,13 +147,13 @@ kerby-pkix-1.0.1.jar
|
||||||
kerby-util-1.0.1.jar
|
kerby-util-1.0.1.jar
|
||||||
kerby-xdr-1.0.1.jar
|
kerby-xdr-1.0.1.jar
|
||||||
kryo-shaded-4.0.2.jar
|
kryo-shaded-4.0.2.jar
|
||||||
kubernetes-client-3.0.0.jar
|
kubernetes-client-4.1.0.jar
|
||||||
kubernetes-model-2.0.0.jar
|
kubernetes-model-4.1.0.jar
|
||||||
leveldbjni-all-1.8.jar
|
leveldbjni-all-1.8.jar
|
||||||
libfb303-0.9.3.jar
|
libfb303-0.9.3.jar
|
||||||
libthrift-0.9.3.jar
|
libthrift-0.9.3.jar
|
||||||
log4j-1.2.17.jar
|
log4j-1.2.17.jar
|
||||||
logging-interceptor-3.8.1.jar
|
logging-interceptor-3.9.1.jar
|
||||||
lz4-java-1.5.0.jar
|
lz4-java-1.5.0.jar
|
||||||
machinist_2.11-0.6.1.jar
|
machinist_2.11-0.6.1.jar
|
||||||
macro-compat_2.11-1.1.1.jar
|
macro-compat_2.11-1.1.1.jar
|
||||||
|
|
|
@ -45,7 +45,8 @@ logs and remains in "completed" state in the Kubernetes API until it's eventuall
|
||||||
|
|
||||||
Note that in the completed state, the driver pod does *not* use any computational or memory resources.
|
Note that in the completed state, the driver pod does *not* use any computational or memory resources.
|
||||||
|
|
||||||
The driver and executor pod scheduling is handled by Kubernetes. It is possible to schedule the
|
The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8, and we are
|
||||||
|
currently running <code>kubernetes-client</code> version <code>4.1.0</code>. Make sure that when you are making infrastructure additions that you are aware of said version. It is possible to schedule the
|
||||||
driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)
|
driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)
|
||||||
using the configuration property for it. It will be possible to use more advanced
|
using the configuration property for it. It will be possible to use more advanced
|
||||||
scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release.
|
scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release.
|
||||||
|
|
|
@ -29,7 +29,7 @@
|
||||||
<name>Spark Project Kubernetes</name>
|
<name>Spark Project Kubernetes</name>
|
||||||
<properties>
|
<properties>
|
||||||
<sbt.project.name>kubernetes</sbt.project.name>
|
<sbt.project.name>kubernetes</sbt.project.name>
|
||||||
<kubernetes.client.version>3.0.0</kubernetes.client.version>
|
<kubernetes.client.version>4.1.0</kubernetes.client.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|
|
@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s
|
||||||
|
|
||||||
import scala.collection.JavaConverters._
|
import scala.collection.JavaConverters._
|
||||||
|
|
||||||
import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time}
|
import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod}
|
||||||
|
|
||||||
import org.apache.spark.{SparkConf, SparkException}
|
import org.apache.spark.{SparkConf, SparkException}
|
||||||
import org.apache.spark.util.Utils
|
import org.apache.spark.util.Utils
|
||||||
|
@ -157,7 +157,7 @@ private[spark] object KubernetesUtils {
|
||||||
}.getOrElse(Seq(("container state", "N/A")))
|
}.getOrElse(Seq(("container state", "N/A")))
|
||||||
}
|
}
|
||||||
|
|
||||||
def formatTime(time: Time): String = {
|
def formatTime(time: String): String = {
|
||||||
if (time != null) time.getTime else "N/A"
|
if (time != null) time else "N/A"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,8 +56,9 @@ private[spark] class MountVolumesFeatureStep(
|
||||||
|
|
||||||
val volumeBuilder = spec.volumeConf match {
|
val volumeBuilder = spec.volumeConf match {
|
||||||
case KubernetesHostPathVolumeConf(hostPath) =>
|
case KubernetesHostPathVolumeConf(hostPath) =>
|
||||||
|
/* "" means that no checks will be performed before mounting the hostPath volume */
|
||||||
new VolumeBuilder()
|
new VolumeBuilder()
|
||||||
.withHostPath(new HostPathVolumeSource(hostPath))
|
.withHostPath(new HostPathVolumeSource(hostPath, ""))
|
||||||
|
|
||||||
case KubernetesPVCVolumeConf(claimName) =>
|
case KubernetesPVCVolumeConf(claimName) =>
|
||||||
new VolumeBuilder()
|
new VolumeBuilder()
|
||||||
|
|
|
@ -20,7 +20,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
|
||||||
|
|
||||||
import scala.collection.JavaConverters._
|
import scala.collection.JavaConverters._
|
||||||
|
|
||||||
import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time}
|
import io.fabric8.kubernetes.api.model.Pod
|
||||||
import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
|
import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
|
||||||
import io.fabric8.kubernetes.client.Watcher.Action
|
import io.fabric8.kubernetes.client.Watcher.Action
|
||||||
|
|
||||||
|
|
|
@ -82,7 +82,7 @@ object ExecutorLifecycleTestUtils {
|
||||||
def deletedExecutor(executorId: Long): Pod = {
|
def deletedExecutor(executorId: Long): Pod = {
|
||||||
new PodBuilder(podWithAttachedContainerForId(executorId))
|
new PodBuilder(podWithAttachedContainerForId(executorId))
|
||||||
.editOrNewMetadata()
|
.editOrNewMetadata()
|
||||||
.withNewDeletionTimestamp("523012521")
|
.withDeletionTimestamp("523012521")
|
||||||
.endMetadata()
|
.endMetadata()
|
||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,7 +29,7 @@
|
||||||
<download-maven-plugin.version>1.3.0</download-maven-plugin.version>
|
<download-maven-plugin.version>1.3.0</download-maven-plugin.version>
|
||||||
<exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
|
<exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
|
||||||
<extraScalaTestArgs></extraScalaTestArgs>
|
<extraScalaTestArgs></extraScalaTestArgs>
|
||||||
<kubernetes-client.version>3.0.0</kubernetes-client.version>
|
<kubernetes-client.version>4.1.0</kubernetes-client.version>
|
||||||
<scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
|
<scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
|
||||||
<scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
|
<scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
|
||||||
<sbt.project.name>kubernetes-integration-tests</sbt.project.name>
|
<sbt.project.name>kubernetes-integration-tests</sbt.project.name>
|
||||||
|
|
Loading…
Reference in a new issue