8c3469009c
### What changes were proposed in this pull request? Currently, `docker-integration-tests` is broken in both JDK8/11. This PR aims to recover JDBC integration test for JDK8/11. ### Why are the changes needed? While SPARK-28737 upgraded `Jersey` to 2.29 for JDK11, `docker-integration-tests` is broken because `com.spotify.docker-client` still depends on `jersey-guava`. The latest `com.spotify.docker-client` also has this problem. - https://mvnrepository.com/artifact/com.spotify/docker-client/5.0.2 -> https://mvnrepository.com/artifact/org.glassfish.jersey.core/jersey-client/2.19 -> https://mvnrepository.com/artifact/org.glassfish.jersey.core/jersey-common/2.19 -> https://mvnrepository.com/artifact/org.glassfish.jersey.bundles.repackaged/jersey-guava/2.19 ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Manual because this is an integration test suite. ``` $ java -version openjdk version "1.8.0_222" OpenJDK Runtime Environment (AdoptOpenJDK)(build 1.8.0_222-b10) OpenJDK 64-Bit Server VM (AdoptOpenJDK)(build 25.222-b10, mixed mode) $ build/mvn install -DskipTests $ build/mvn -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 test ``` ``` $ java -version openjdk version "11.0.5" 2019-10-15 OpenJDK Runtime Environment AdoptOpenJDK (build 11.0.5+10) OpenJDK 64-Bit Server VM AdoptOpenJDK (build 11.0.5+10, mixed mode) $ build/mvn install -DskipTests $ build/mvn -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 test ``` **BEFORE** ``` *** RUN ABORTED *** com.spotify.docker.client.exceptions.DockerException: java.util.concurrent.ExecutionException: javax.ws.rs.ProcessingException: java.lang.NoClassDefFoundError: jersey/repackaged/com/google/common/util/concurrent/MoreExecutors at com.spotify.docker.client.DefaultDockerClient.propagate(DefaultDockerClient.java:1607) at com.spotify.docker.client.DefaultDockerClient.request(DefaultDockerClient.java:1538) at com.spotify.docker.client.DefaultDockerClient.ping(DefaultDockerClient.java:387) at org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite.beforeAll(DockerJDBCIntegrationSuite.scala:81) ``` **AFTER** ``` Run completed in 47 seconds, 999 milliseconds. Total number of tests run: 30 Suites: completed 6, aborted 0 Tests: succeeded 30, failed 0, canceled 0, ignored 6, pending 0 All tests passed. ``` Closes #26203 from dongjoon-hyun/SPARK-29546. Authored-by: Dongjoon Hyun <dhyun@apple.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
169 lines
6 KiB
XML
169 lines
6 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent_2.12</artifactId>
|
|
<version>3.0.0-SNAPSHOT</version>
|
|
<relativePath>../../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>spark-docker-integration-tests_2.12</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Spark Project Docker Integration Tests</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<properties>
|
|
<sbt.project.name>docker-integration-tests</sbt.project.name>
|
|
</properties>
|
|
|
|
<repositories>
|
|
<repository>
|
|
<id>db</id>
|
|
<url>https://app.camunda.com/nexus/content/repositories/public/</url>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
<checksumPolicy>warn</checksumPolicy>
|
|
</releases>
|
|
</repository>
|
|
</repositories>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>com.spotify</groupId>
|
|
<artifactId>docker-client</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpcore</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Necessary in order to avoid errors in log messages: -->
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<version>18.0</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Although SPARK-28737 upgraded Jersey to 2.29 for JDK11, 'com.spotify.docker-client' still
|
|
uses this repackaged 'jersey-guava'. We add this back for JDK8/JDK11 testing. -->
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.bundles.repackaged</groupId>
|
|
<artifactId>jersey-guava</artifactId>
|
|
<version>2.25.1</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>mysql</groupId>
|
|
<artifactId>mysql-connector-java</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.postgresql</groupId>
|
|
<artifactId>postgresql</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Oracle ojdbc jar, used for oracle integration suite for docker testing.
|
|
See https://github.com/apache/spark/pull/11306 for background on why we need
|
|
to use a an ojdbc jar for the testcase. The maven dependency here is commented
|
|
because currently the maven repository does not contain the ojdbc jar mentioned.
|
|
Once the jar is available in maven, this could be uncommented. -->
|
|
<dependency>
|
|
<groupId>com.oracle</groupId>
|
|
<artifactId>ojdbc6</artifactId>
|
|
<version>11.2.0.1.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!-- DB2 JCC driver manual installation instructions
|
|
|
|
You can build this datasource if you:
|
|
1) have the DB2 artifacts installed in a local repo and supply the URL:
|
|
-Dmaven.repo.drivers=http://my.local.repo
|
|
|
|
2) have a copy of the DB2 JCC driver and run the following commands :
|
|
mvn install:install-file -Dfile=${path to db2jcc4.jar} \
|
|
-DgroupId=com.ibm.db2 \
|
|
-DartifactId=db2jcc4 \
|
|
-Dversion=10.5 \
|
|
-Dpackaging=jar
|
|
|
|
Note: IBM DB2 JCC driver is available for download at
|
|
http://www-01.ibm.com/support/docview.wss?uid=swg21363866
|
|
-->
|
|
<dependency>
|
|
<groupId>com.ibm.db2.jcc</groupId>
|
|
<artifactId>db2jcc4</artifactId>
|
|
<version>10.5.0.5</version>
|
|
<type>jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.microsoft.sqlserver</groupId>
|
|
<artifactId>mssql-jdbc</artifactId>
|
|
<version>7.2.1.jre8</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
</project>
|