[SPARK-32568][BUILD][SS] Upgrade Kafka to 2.6.0
### What changes were proposed in this pull request? This PR aims to update Kafka client library to 2.6.0 for Apache Spark 3.1.0. ### Why are the changes needed? This will bring client-side bug fixes like KAFKA-10134 and KAFKA-10223. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the existing tests. Closes #29386 from dongjoon-hyun/SPARK-32568. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
parent
5b8444af0d
commit
eb74d55fb5
|
@ -395,7 +395,7 @@ class KafkaTestUtils(
|
|||
}
|
||||
|
||||
def getAllTopicsAndPartitionSize(): Seq[(String, Int)] = {
|
||||
zkClient.getPartitionsForTopics(zkClient.getAllTopicsInCluster).mapValues(_.size).toSeq
|
||||
zkClient.getPartitionsForTopics(zkClient.getAllTopicsInCluster()).mapValues(_.size).toSeq
|
||||
}
|
||||
|
||||
/** Create a Kafka topic and wait until it is propagated to the whole cluster */
|
||||
|
@ -587,7 +587,7 @@ class KafkaTestUtils(
|
|||
}), s"checkpoint for topic $topic still exists")
|
||||
// ensure the topic is gone
|
||||
assert(
|
||||
!zkClient.getAllTopicsInCluster.contains(topic),
|
||||
!zkClient.getAllTopicsInCluster().contains(topic),
|
||||
s"topic $topic still exists on zookeeper")
|
||||
}
|
||||
|
||||
|
|
2
pom.xml
2
pom.xml
|
@ -133,7 +133,7 @@
|
|||
<!-- Version used for internal directory structure -->
|
||||
<hive.version.short>2.3</hive.version.short>
|
||||
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
|
||||
<kafka.version>2.5.0</kafka.version>
|
||||
<kafka.version>2.6.0</kafka.version>
|
||||
<derby.version>10.12.1.1</derby.version>
|
||||
<parquet.version>1.10.1</parquet.version>
|
||||
<orc.version>1.5.10</orc.version>
|
||||
|
|
Loading…
Reference in a new issue