d5b7eed12f
The Pyspark Kinesis tests are failing, at least in master:
```
======================================================================
ERROR: test_kinesis_stream (pyspark.streaming.tests.test_kinesis.KinesisStreamTests)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/jenkins/workspace/SparkPullRequestBuilder2/python/pyspark/streaming/tests/test_kinesis.py", line 44, in test_kinesis_stream
kinesisTestUtils = self.ssc._jvm.org.apache.spark.streaming.kinesis.KinesisTestUtils(2)
File "/home/jenkins/workspace/SparkPullRequestBuilder2/python/lib/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1554, in __call__
answer, self._gateway_client, None, self._fqn)
File "/home/jenkins/workspace/SparkPullRequestBuilder2/python/lib/py4j-0.10.8.1-src.zip/py4j/protocol.py", line 328, in get_return_value
format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling None.org.apache.spark.streaming.kinesis.KinesisTestUtils.
: java.lang.NoSuchMethodError: com.amazonaws.regions.Region.getAvailableEndpoints()Ljava/util/Collection;
at org.apache.spark.streaming.kinesis.KinesisTestUtils$.$anonfun$getRegionNameByEndpoint$1(KinesisTestUtils.scala:211)
at org.apache.spark.streaming.kinesis.KinesisTestUtils$.$anonfun$getRegionNameByEndpoint$1$adapted(KinesisTestUtils.scala:211)
at scala.collection.Iterator.find(Iterator.scala:993)
at scala.collection.Iterator.find$(Iterator.scala:990)
at scala.collection.AbstractIterator.find(Iterator.scala:1429)
at scala.collection.IterableLike.find(IterableLike.scala:81)
at scala.collection.IterableLike.find$(IterableLike.scala:80)
at scala.collection.AbstractIterable.find(Iterable.scala:56)
at org.apache.spark.streaming.kinesis.KinesisTestUtils$.getRegionNameByEndpoint(KinesisTestUtils.scala:211)
at org.apache.spark.streaming.kinesis.KinesisTestUtils.<init>(KinesisTestUtils.scala:46)
...
```
The non-Python Kinesis tests are fine though. It turns out that this is because Pyspark tests use the output of the Spark assembly, and it pulls in `hadoop-cloud`, which in turn pulls in an old AWS Java SDK.
Per Steve Loughran (below), it seems like we can just resolve this by excluding the aws-java-sdk dependency. See the attached PR for some more detail about the debugging and other options.
See https://github.com/apache/spark/pull/25558#issuecomment-524042709
Closes #25559 from srowen/KinesisTest.
Authored-by: Sean Owen <sean.owen@databricks.com>
Signed-off-by: Sean Owen <sean.owen@databricks.com>
298 lines
10 KiB
XML
298 lines
10 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent_2.12</artifactId>
|
|
<version>3.0.0-SNAPSHOT</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>spark-hadoop-cloud_2.12</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Spark Project Hadoop Cloud Integration</name>
|
|
<description>
|
|
Contains Hadoop JARs and transitive dependencies needed to interact with cloud infrastructures.
|
|
</description>
|
|
<properties>
|
|
<sbt.project.name>hadoop-cloud</sbt.project.name>
|
|
</properties>
|
|
|
|
<build>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
</build>
|
|
|
|
<dependencies>
|
|
<!--used during compilation but not exported as transitive dependencies-->
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<!--
|
|
the AWS module pulls in jackson; its transitive dependencies can create
|
|
intra-jackson-module version problems.
|
|
-->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-aws</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</exclusion>
|
|
<!-- Keep old SDK out of the assembly to avoid conflict with Kinesis module -->
|
|
<exclusion>
|
|
<groupId>com.amazonaws</groupId>
|
|
<artifactId>aws-java-sdk</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-openstack</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-all</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!--
|
|
Add joda time to ensure that anything downstream which doesn't pull in spark-hive
|
|
gets the correct joda time artifact, so doesn't have auth failures on later Java 8 JVMs
|
|
-->
|
|
<dependency>
|
|
<groupId>joda-time</groupId>
|
|
<artifactId>joda-time</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<!-- explicitly declare the jackson artifacts desired -->
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
|
<artifactId>jackson-dataformat-cbor</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
</dependency>
|
|
<!--Explicit declaration to force in Spark version into transitive dependencies -->
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<!--Explicit declaration to force in Spark version into transitive dependencies -->
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpcore</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-azure</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<profiles>
|
|
|
|
<!--
|
|
Hadoop 3 simplifies the classpath, and adds a new committer base class which
|
|
enables store-specific committers.
|
|
-->
|
|
<profile>
|
|
<id>hadoop-3.2</id>
|
|
<properties>
|
|
<extra.source.dir>src/hadoop-3/main/scala</extra.source.dir>
|
|
<extra.testsource.dir>src/hadoop-3/test/scala</extra.testsource.dir>
|
|
</properties>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>add-scala-sources</id>
|
|
<phase>generate-sources</phase>
|
|
<goals>
|
|
<goal>add-source</goal>
|
|
</goals>
|
|
<configuration>
|
|
<sources>
|
|
<source>${extra.source.dir}</source>
|
|
</sources>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>add-scala-test-sources</id>
|
|
<phase>generate-test-sources</phase>
|
|
<goals>
|
|
<goal>add-test-source</goal>
|
|
</goals>
|
|
<configuration>
|
|
<sources>
|
|
<source>${extra.testsource.dir}</source>
|
|
</sources>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
<dependencies>
|
|
<!--
|
|
There's now a hadoop-cloud-storage which transitively pulls in the store JARs,
|
|
but it still needs some selective exclusion across versions, especially 3.0.x.
|
|
-->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-cloud-storage</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!--
|
|
The jetty declarations are made
|
|
(a) to keep that jetty-util-ajax version in sync with the rest of Spark.
|
|
(b) to minimise the effects which Spark's jetty shading has on the
|
|
availability of the jetty JARs on for hadoop-azure, which depends
|
|
on them.
|
|
-->
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-util-ajax</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
|
|
</profiles>
|
|
|
|
</project>
|