[SPARK-27812][K8S] Bump K8S client version to 4.6.1
### What changes were proposed in this pull request? Updated kubernetes client. ### Why are the changes needed? https://issues.apache.org/jira/browse/SPARK-27812 https://issues.apache.org/jira/browse/SPARK-27927 We need this fix https://github.com/fabric8io/kubernetes-client/pull/1768 that was released on version 4.6 of the client. The root cause of the problem is better explained in https://github.com/apache/spark/pull/25785 ### Does this PR introduce any user-facing change? Nope, it should be transparent to users ### How was this patch tested? This patch was tested manually using a simple pyspark job ```python from pyspark.sql import SparkSession if __name__ == '__main__': spark = SparkSession.builder.getOrCreate() ``` The expected behaviour of this "job" is that both python's and jvm's process exit automatically after the main runs. This is the case for spark versions <= 2.4. On version 2.4.3, the jvm process hangs because there's a non daemon thread running ``` "OkHttp WebSocket https://10.96.0.1/..." #121 prio=5 os_prio=0 tid=0x00007fb27c005800 nid=0x24b waiting on condition [0x00007fb300847000] "OkHttp WebSocket https://10.96.0.1/..." #117 prio=5 os_prio=0 tid=0x00007fb28c004000 nid=0x247 waiting on condition [0x00007fb300e4b000] ``` This is caused by a bug on `kubernetes-client` library, which is fixed on the version that we are upgrading to. When the mentioned job is run with this patch applied, the behaviour from spark <= 2.4.3 is restored and both processes terminate successfully Closes #26093 from igorcalabria/k8s-client-update. Authored-by: igor.calabria <igor.calabria@ubee.in> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
parent
100fc58da5
commit
78bdcfade1
|
@ -107,7 +107,6 @@ jakarta.ws.rs-api-2.1.5.jar
|
|||
jakarta.xml.bind-api-2.3.2.jar
|
||||
janino-3.0.15.jar
|
||||
javassist-3.22.0-CR2.jar
|
||||
javax.el-3.0.1-b11.jar
|
||||
javax.inject-1.jar
|
||||
javax.servlet-api-3.1.0.jar
|
||||
javolution-5.5.1.jar
|
||||
|
@ -138,9 +137,9 @@ jsr305-3.0.0.jar
|
|||
jta-1.1.jar
|
||||
jul-to-slf4j-1.7.16.jar
|
||||
kryo-shaded-4.0.2.jar
|
||||
kubernetes-client-4.4.2.jar
|
||||
kubernetes-model-4.4.2.jar
|
||||
kubernetes-model-common-4.4.2.jar
|
||||
kubernetes-client-4.6.1.jar
|
||||
kubernetes-model-4.6.1.jar
|
||||
kubernetes-model-common-4.6.1.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.3.jar
|
||||
libthrift-0.12.0.jar
|
||||
|
@ -158,8 +157,8 @@ minlog-1.3.0.jar
|
|||
netty-all-4.1.42.Final.jar
|
||||
objenesis-2.5.1.jar
|
||||
okapi-shade-0.4.2.jar
|
||||
okhttp-3.8.1.jar
|
||||
okio-1.13.0.jar
|
||||
okhttp-3.12.0.jar
|
||||
okio-1.15.0.jar
|
||||
opencsv-2.3.jar
|
||||
orc-core-1.5.6-nohive.jar
|
||||
orc-mapreduce-1.5.6-nohive.jar
|
||||
|
|
|
@ -120,7 +120,6 @@ jakarta.ws.rs-api-2.1.5.jar
|
|||
jakarta.xml.bind-api-2.3.2.jar
|
||||
janino-3.0.15.jar
|
||||
javassist-3.22.0-CR2.jar
|
||||
javax.el-3.0.1-b11.jar
|
||||
javax.inject-1.jar
|
||||
javax.jdo-3.2.0-m3.jar
|
||||
javax.servlet-api-3.1.0.jar
|
||||
|
@ -168,9 +167,9 @@ kerby-pkix-1.0.1.jar
|
|||
kerby-util-1.0.1.jar
|
||||
kerby-xdr-1.0.1.jar
|
||||
kryo-shaded-4.0.2.jar
|
||||
kubernetes-client-4.4.2.jar
|
||||
kubernetes-model-4.4.2.jar
|
||||
kubernetes-model-common-4.4.2.jar
|
||||
kubernetes-client-4.6.1.jar
|
||||
kubernetes-model-4.6.1.jar
|
||||
kubernetes-model-common-4.6.1.jar
|
||||
leveldbjni-all-1.8.jar
|
||||
libfb303-0.9.3.jar
|
||||
libthrift-0.12.0.jar
|
||||
|
@ -191,8 +190,8 @@ nimbus-jose-jwt-4.41.1.jar
|
|||
objenesis-2.5.1.jar
|
||||
okapi-shade-0.4.2.jar
|
||||
okhttp-2.7.5.jar
|
||||
okhttp-3.8.1.jar
|
||||
okio-1.13.0.jar
|
||||
okhttp-3.12.0.jar
|
||||
okio-1.15.0.jar
|
||||
opencsv-2.3.jar
|
||||
orc-core-1.5.6.jar
|
||||
orc-mapreduce-1.5.6.jar
|
||||
|
|
|
@ -29,7 +29,7 @@
|
|||
<name>Spark Project Kubernetes</name>
|
||||
<properties>
|
||||
<sbt.project.name>kubernetes</sbt.project.name>
|
||||
<kubernetes.client.version>4.4.2</kubernetes.client.version>
|
||||
<kubernetes.client.version>4.6.1</kubernetes.client.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -88,12 +88,6 @@
|
|||
</dependency>
|
||||
<!-- End of shaded deps. -->
|
||||
|
||||
<dependency>
|
||||
<groupId>com.squareup.okhttp3</groupId>
|
||||
<artifactId>okhttp</artifactId>
|
||||
<version>3.8.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
|
|
|
@ -29,7 +29,7 @@
|
|||
<download-maven-plugin.version>1.3.0</download-maven-plugin.version>
|
||||
<exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
|
||||
<extraScalaTestArgs></extraScalaTestArgs>
|
||||
<kubernetes-client.version>4.4.2</kubernetes-client.version>
|
||||
<kubernetes-client.version>4.6.1</kubernetes-client.version>
|
||||
<scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
|
||||
<scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
|
||||
<sbt.project.name>kubernetes-integration-tests</sbt.project.name>
|
||||
|
|
Loading…
Reference in a new issue