dbb824125d
## What changes were proposed in this pull request? `HiveExternalCatalog` is a semi-public interface. When creating tables, `HiveExternalCatalog` converts the table metadata to hive table format and save into hive metastore. It's very import to guarantee backward compatibility here, i.e., tables created by previous Spark versions should still be readable in newer Spark versions. Previously we find backward compatibility issues manually, which is really easy to miss bugs. This PR introduces a test framework to automatically test `HiveExternalCatalog` backward compatibility, by downloading Spark binaries with different versions, and create tables with these Spark versions, and read these tables with current Spark version. ## How was this patch tested? test-only change Author: Wenchen Fan <wenchen@databricks.com> Closes #19148 from cloud-fan/test.
259 lines
8.4 KiB
XML
259 lines
8.4 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent_2.11</artifactId>
|
|
<version>2.3.0-SNAPSHOT</version>
|
|
<relativePath>../../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>spark-hive_2.11</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Spark Project Hive</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<properties>
|
|
<sbt.project.name>hive</sbt.project.name>
|
|
</properties>
|
|
|
|
<dependencies>
|
|
<!-- Added for Hive Parquet SerDe -->
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>parquet-hadoop-bundle</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!--
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.protobuf</groupId>
|
|
<artifactId>protobuf-java</artifactId>
|
|
<version>${protobuf.version}</version>
|
|
</dependency>
|
|
-->
|
|
<!--
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</dependency>
|
|
-->
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-exec</artifactId>
|
|
<!--
|
|
<classifier>core</classifier>
|
|
-->
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
</dependency>
|
|
<!--
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</dependency>
|
|
-->
|
|
<!-- hive-serde already depends on avro, but this brings in customized config of avro deps from parent -->
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
</dependency>
|
|
<!-- use the build matching the hadoop api of avro-mapred (i.e. no classifier for hadoop 1 API,
|
|
hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of org.spark-project.hive:hive-serde -->
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-mapred</artifactId>
|
|
<classifier>${avro.mapred.classifier}</classifier>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-httpclient</groupId>
|
|
<artifactId>commons-httpclient</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.calcite</groupId>
|
|
<artifactId>calcite-avatica</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.calcite</groupId>
|
|
<artifactId>calcite-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
</dependency>
|
|
<!-- transitive dependencies of hive-exec-core doesn't declare -->
|
|
<dependency>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>joda-time</groupId>
|
|
<artifactId>joda-time</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.jodd</groupId>
|
|
<artifactId>jodd-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.datanucleus</groupId>
|
|
<artifactId>datanucleus-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.derby</groupId>
|
|
<artifactId>derby</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-compiler</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalacheck</groupId>
|
|
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
<profiles>
|
|
<profile>
|
|
<id>hive</id>
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
<version>3.0.0</version>
|
|
<executions>
|
|
<execution>
|
|
<id>add-scala-test-sources</id>
|
|
<phase>generate-test-sources</phase>
|
|
<goals>
|
|
<goal>add-test-source</goal>
|
|
</goals>
|
|
<configuration>
|
|
<sources>
|
|
<source>compatibility/src/test/scala</source>
|
|
</sources>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</profile>
|
|
</profiles>
|
|
|
|
<build>
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
<configuration>
|
|
<!-- Specially disable assertions since some Hive tests fail them -->
|
|
<argLine>-da -Xmx3g -XX:ReservedCodeCacheSize=${CodeCacheSize}</argLine>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-enforcer-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>enforce-versions</id>
|
|
<goals>
|
|
<goal>enforce</goal>
|
|
</goals>
|
|
<configuration>
|
|
<rules>
|
|
<bannedDependencies>
|
|
<excludes combine.children="append">
|
|
<exclude>*:hive-cli</exclude>
|
|
</excludes>
|
|
</bannedDependencies>
|
|
</rules>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</project>
|