### What changes were proposed in this pull request? This PR fix the issue that spark-shell doesn't work if it's built with `sbt package` (without any profiles specified). It's due to hadoop-client-runtime.jar isn't copied to assembly/target/scala-2.12/jars. ``` $ bin/spark-shell Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/shaded/com/ctc/wstx/io/InputBootstrapper at org.apache.spark.deploy.SparkHadoopUtil$.newConfiguration(SparkHadoopUtil.scala:426) at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$2(SparkSubmit.scala:342) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:342) at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:877) at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.shaded.com.ctc.wstx.io.InputBootstrapper at java.net.URLClassLoader.findClass(URLClassLoader.java:382) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ``` ### Why are the changes needed? This is a bug. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Ran spark-shell and confirmed it works. Closes #30250 from sarutak/copy-runtime-sbt. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
3352 lines
119 KiB
XML
3352 lines
119 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
~ this work for additional information regarding copyright ownership.
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
~ the License. You may obtain a copy of the License at
|
|
~
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
~
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
~ See the License for the specific language governing permissions and
|
|
~ limitations under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache</groupId>
|
|
<artifactId>apache</artifactId>
|
|
<version>18</version>
|
|
</parent>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-parent_2.12</artifactId>
|
|
<version>3.1.0-SNAPSHOT</version>
|
|
<packaging>pom</packaging>
|
|
<name>Spark Project Parent POM</name>
|
|
<url>http://spark.apache.org/</url>
|
|
<licenses>
|
|
<license>
|
|
<name>Apache 2.0 License</name>
|
|
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
|
|
<distribution>repo</distribution>
|
|
</license>
|
|
</licenses>
|
|
<scm>
|
|
<connection>scm:git:git@github.com:apache/spark.git</connection>
|
|
<developerConnection>scm:git:https://gitbox.apache.org/repos/asf/spark.git</developerConnection>
|
|
<url>scm:git:git@github.com:apache/spark.git</url>
|
|
<tag>HEAD</tag>
|
|
</scm>
|
|
<developers>
|
|
<developer>
|
|
<id>matei</id>
|
|
<name>Matei Zaharia</name>
|
|
<email>matei.zaharia@gmail.com</email>
|
|
<url>http://www.cs.berkeley.edu/~matei</url>
|
|
<organization>Apache Software Foundation</organization>
|
|
<organizationUrl>http://spark.apache.org</organizationUrl>
|
|
</developer>
|
|
</developers>
|
|
<issueManagement>
|
|
<system>JIRA</system>
|
|
<url>https://issues.apache.org/jira/browse/SPARK</url>
|
|
</issueManagement>
|
|
|
|
<mailingLists>
|
|
<mailingList>
|
|
<name>Dev Mailing List</name>
|
|
<post>dev@spark.apache.org</post>
|
|
<subscribe>dev-subscribe@spark.apache.org</subscribe>
|
|
<unsubscribe>dev-unsubscribe@spark.apache.org</unsubscribe>
|
|
</mailingList>
|
|
|
|
<mailingList>
|
|
<name>User Mailing List</name>
|
|
<post>user@spark.apache.org</post>
|
|
<subscribe>user-subscribe@spark.apache.org</subscribe>
|
|
<unsubscribe>user-unsubscribe@spark.apache.org</unsubscribe>
|
|
</mailingList>
|
|
|
|
<mailingList>
|
|
<name>Commits Mailing List</name>
|
|
<post>commits@spark.apache.org</post>
|
|
<subscribe>commits-subscribe@spark.apache.org</subscribe>
|
|
<unsubscribe>commits-unsubscribe@spark.apache.org</unsubscribe>
|
|
</mailingList>
|
|
</mailingLists>
|
|
|
|
<modules>
|
|
<module>common/sketch</module>
|
|
<module>common/kvstore</module>
|
|
<module>common/network-common</module>
|
|
<module>common/network-shuffle</module>
|
|
<module>common/unsafe</module>
|
|
<module>common/tags</module>
|
|
<module>core</module>
|
|
<module>graphx</module>
|
|
<module>mllib</module>
|
|
<module>mllib-local</module>
|
|
<module>tools</module>
|
|
<module>streaming</module>
|
|
<module>sql/catalyst</module>
|
|
<module>sql/core</module>
|
|
<module>sql/hive</module>
|
|
<module>assembly</module>
|
|
<module>examples</module>
|
|
<module>repl</module>
|
|
<module>launcher</module>
|
|
<module>external/kafka-0-10-token-provider</module>
|
|
<module>external/kafka-0-10</module>
|
|
<module>external/kafka-0-10-assembly</module>
|
|
<module>external/kafka-0-10-sql</module>
|
|
<module>external/avro</module>
|
|
<!-- See additional modules enabled by profiles below -->
|
|
</modules>
|
|
|
|
<properties>
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
|
<java.version>1.8</java.version>
|
|
<maven.compiler.source>${java.version}</maven.compiler.source>
|
|
<maven.compiler.target>${java.version}</maven.compiler.target>
|
|
<maven.version>3.6.3</maven.version>
|
|
<exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
|
|
<sbt.project.name>spark</sbt.project.name>
|
|
<slf4j.version>1.7.30</slf4j.version>
|
|
<log4j.version>1.2.17</log4j.version>
|
|
<hadoop.version>3.2.0</hadoop.version>
|
|
<protobuf.version>2.5.0</protobuf.version>
|
|
<yarn.version>${hadoop.version}</yarn.version>
|
|
<zookeeper.version>3.4.14</zookeeper.version>
|
|
<curator.version>2.13.0</curator.version>
|
|
<hive.group>org.apache.hive</hive.group>
|
|
<hive.classifier>core</hive.classifier>
|
|
<!-- Version used in Maven Hive dependency -->
|
|
<hive.version>2.3.7</hive.version>
|
|
<hive23.version>2.3.7</hive23.version>
|
|
<!-- Version used for internal directory structure -->
|
|
<hive.version.short>2.3</hive.version.short>
|
|
<!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
|
|
<kafka.version>2.6.0</kafka.version>
|
|
<derby.version>10.12.1.1</derby.version>
|
|
<parquet.version>1.10.1</parquet.version>
|
|
<orc.version>1.5.12</orc.version>
|
|
<jetty.version>9.4.28.v20200408</jetty.version>
|
|
<javaxservlet.version>3.1.0</javaxservlet.version>
|
|
<chill.version>0.9.5</chill.version>
|
|
<ivy.version>2.4.0</ivy.version>
|
|
<oro.version>2.0.8</oro.version>
|
|
<!--
|
|
If you changes codahale.metrics.version, you also need to change
|
|
the link to metrics.dropwizard.io in docs/monitoring.md.
|
|
-->
|
|
<codahale.metrics.version>4.1.1</codahale.metrics.version>
|
|
<avro.version>1.8.2</avro.version>
|
|
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
|
|
<aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
|
|
<!-- Should be consistent with Kinesis client dependency -->
|
|
<aws.java.sdk.version>1.11.655</aws.java.sdk.version>
|
|
<!-- the producer is used in tests -->
|
|
<aws.kinesis.producer.version>0.12.8</aws.kinesis.producer.version>
|
|
<!-- org.apache.httpcomponents/httpclient-->
|
|
<commons.httpclient.version>4.5.6</commons.httpclient.version>
|
|
<commons.httpcore.version>4.4.12</commons.httpcore.version>
|
|
<!-- commons-httpclient/commons-httpclient-->
|
|
<httpclient.classic.version>3.1</httpclient.classic.version>
|
|
<commons.math3.version>3.4.1</commons.math3.version>
|
|
<!-- managed up from 3.2.1 for SPARK-11652 -->
|
|
<commons.collections.version>3.2.2</commons.collections.version>
|
|
<scala.version>2.12.10</scala.version>
|
|
<scala.binary.version>2.12</scala.binary.version>
|
|
<scalatest-maven-plugin.version>2.0.0</scalatest-maven-plugin.version>
|
|
<scalafmt.parameters>--test</scalafmt.parameters>
|
|
<!-- for now, not running scalafmt as part of default verify pipeline -->
|
|
<scalafmt.skip>true</scalafmt.skip>
|
|
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
|
|
<fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
|
|
<snappy.version>1.1.8</snappy.version>
|
|
<netlib.java.version>1.1.2</netlib.java.version>
|
|
<commons-codec.version>1.10</commons-codec.version>
|
|
<commons-io.version>2.5</commons-io.version>
|
|
<!-- org.apache.commons/commons-lang/-->
|
|
<commons-lang2.version>2.6</commons-lang2.version>
|
|
<!-- org.apache.commons/commons-lang3/-->
|
|
<commons-lang3.version>3.10</commons-lang3.version>
|
|
<!-- org.apache.commons/commons-pool2/-->
|
|
<commons-pool2.version>2.6.2</commons-pool2.version>
|
|
<datanucleus-core.version>4.1.17</datanucleus-core.version>
|
|
<guava.version>14.0.1</guava.version>
|
|
<janino.version>3.0.16</janino.version>
|
|
<jersey.version>2.30</jersey.version>
|
|
<joda.version>2.10.5</joda.version>
|
|
<jodd.version>3.5.2</jodd.version>
|
|
<jsr305.version>3.0.0</jsr305.version>
|
|
<libthrift.version>0.12.0</libthrift.version>
|
|
<antlr4.version>4.7.1</antlr4.version>
|
|
<jpam.version>1.1</jpam.version>
|
|
<selenium.version>3.141.59</selenium.version>
|
|
<htmlunit.version>2.40.0</htmlunit.version>
|
|
<!--
|
|
Managed up from older version from Avro; sync with jackson-module-paranamer dependency version
|
|
-->
|
|
<paranamer.version>2.8</paranamer.version>
|
|
<maven-antrun.version>1.8</maven-antrun.version>
|
|
<commons-crypto.version>1.0.0</commons-crypto.version>
|
|
<!--
|
|
If you are changing Arrow version specification, please check
|
|
./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too.
|
|
-->
|
|
<arrow.version>1.0.1</arrow.version>
|
|
<!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
|
|
<leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
|
|
|
|
<test.java.home>${java.home}</test.java.home>
|
|
|
|
<!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
|
|
<test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
|
|
<test.exclude.tags></test.exclude.tags>
|
|
<test.include.tags></test.include.tags>
|
|
|
|
<!-- Package to use when relocating shaded classes. -->
|
|
<spark.shade.packageName>org.sparkproject</spark.shade.packageName>
|
|
|
|
<!-- Modules that copy jars to the build directory should do so under this location. -->
|
|
<jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
|
|
|
|
<!-- Allow modules to enable / disable certain build plugins easily. -->
|
|
<build.testJarPhase>prepare-package</build.testJarPhase>
|
|
<build.copyDependenciesPhase>none</build.copyDependenciesPhase>
|
|
|
|
<!--
|
|
Dependency scopes that can be overridden by enabling certain profiles. These profiles are
|
|
declared in the projects that build assemblies.
|
|
|
|
For other projects the scope should remain as "compile", otherwise they are not available
|
|
during compilation if the dependency is transivite (e.g. "graphx/" depending on "core/" and
|
|
needing Hadoop classes in the classpath to compile).
|
|
-->
|
|
<hadoop.deps.scope>compile</hadoop.deps.scope>
|
|
<hive.deps.scope>compile</hive.deps.scope>
|
|
<hive.storage.version>2.7.2</hive.storage.version>
|
|
<hive.storage.scope>compile</hive.storage.scope>
|
|
<hive.common.scope>compile</hive.common.scope>
|
|
<hive.llap.scope>compile</hive.llap.scope>
|
|
<hive.serde.scope>compile</hive.serde.scope>
|
|
<hive.shims.scope>compile</hive.shims.scope>
|
|
<orc.deps.scope>compile</orc.deps.scope>
|
|
<parquet.deps.scope>compile</parquet.deps.scope>
|
|
<parquet.test.deps.scope>test</parquet.test.deps.scope>
|
|
|
|
<!--
|
|
These default to Hadoop 3.x shaded client/minicluster jars, but are switched to hadoop-client
|
|
when the Hadoop profile is hadoop-2.7, because these are only available in 3.x. Note that,
|
|
as result we have to include the same hadoop-client dependency multiple times in hadoop-2.7.
|
|
-->
|
|
<hadoop-client-api.artifact>hadoop-client-api</hadoop-client-api.artifact>
|
|
<hadoop-client-runtime.artifact>hadoop-client-runtime</hadoop-client-runtime.artifact>
|
|
<hadoop-client-minicluster.artifact>hadoop-client-minicluster</hadoop-client-minicluster.artifact>
|
|
|
|
<!--
|
|
Overridable test home. So that you can call individual pom files directly without
|
|
things breaking.
|
|
-->
|
|
<spark.test.home>${session.executionRootDirectory}</spark.test.home>
|
|
<spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
|
|
|
|
<CodeCacheSize>1g</CodeCacheSize>
|
|
<!-- Needed for consistent times -->
|
|
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
|
|
</properties>
|
|
<repositories>
|
|
<repository>
|
|
<id>gcs-maven-central-mirror</id>
|
|
<!--
|
|
Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
|
|
See https://storage-download.googleapis.com/maven-central/index.html
|
|
-->
|
|
<name>GCS Maven Central mirror</name>
|
|
<url>https://maven-central.storage-download.googleapis.com/maven2/</url>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>false</enabled>
|
|
</snapshots>
|
|
</repository>
|
|
<repository>
|
|
<!--
|
|
This is used as a fallback when the first try fails.
|
|
-->
|
|
<id>central</id>
|
|
<name>Maven Repository</name>
|
|
<url>https://repo.maven.apache.org/maven2</url>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>false</enabled>
|
|
</snapshots>
|
|
</repository>
|
|
</repositories>
|
|
<pluginRepositories>
|
|
<pluginRepository>
|
|
<id>gcs-maven-central-mirror</id>
|
|
<!--
|
|
Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
|
|
See https://storage-download.googleapis.com/maven-central/index.html
|
|
-->
|
|
<name>GCS Maven Central mirror</name>
|
|
<url>https://maven-central.storage-download.googleapis.com/maven2/</url>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>false</enabled>
|
|
</snapshots>
|
|
</pluginRepository>
|
|
<pluginRepository>
|
|
<id>central</id>
|
|
<url>https://repo.maven.apache.org/maven2</url>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>false</enabled>
|
|
</snapshots>
|
|
</pluginRepository>
|
|
</pluginRepositories>
|
|
<dependencies>
|
|
<!--
|
|
This is a dummy dependency that is used to trigger the maven-shade plugin so that Spark's
|
|
published POMs are flattened and do not contain variables. Without this dependency, some
|
|
subprojects' published POMs would contain variables like ${scala.binary.version} that will
|
|
be substituted according to the default properties instead of the ones determined by the
|
|
profiles that were active during publishing, causing the Scala 2.10 build's POMs to have 2.11
|
|
dependencies due to the incorrect substitutions. By ensuring that maven-shade runs for all
|
|
subprojects, we eliminate this problem because the substitutions are baked into the final POM.
|
|
|
|
For more details, see SPARK-3812 and MNG-2971.
|
|
-->
|
|
<dependency>
|
|
<groupId>org.spark-project.spark</groupId>
|
|
<artifactId>unused</artifactId>
|
|
<version>1.0.0</version>
|
|
</dependency>
|
|
<!--
|
|
This is needed by the scalatest plugin, and so is declared here to be available in
|
|
all child modules, just as scalatest is run in all children
|
|
-->
|
|
<dependency>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-scalacheck_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-mockito_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-selenium_${scala.binary.version}</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.novocode</groupId>
|
|
<artifactId>junit-interface</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
<dependencyManagement>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-tags_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill_${scala.binary.version}</artifactId>
|
|
<version>${chill.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.twitter</groupId>
|
|
<artifactId>chill-java</artifactId>
|
|
<version>${chill.version}</version>
|
|
</dependency>
|
|
<!-- This artifact is a shaded version of ASM 7.x. The POM that was used to produce this
|
|
is at https://github.com/apache/geronimo-xbean/tree/trunk/xbean-asm7-shaded
|
|
For context on why we shade ASM, see SPARK-782 and SPARK-6152. -->
|
|
<dependency>
|
|
<groupId>org.apache.xbean</groupId>
|
|
<artifactId>xbean-asm7-shaded</artifactId>
|
|
<version>4.15</version>
|
|
</dependency>
|
|
|
|
<!-- Shaded deps marked as provided. These are promoted to compile scope
|
|
in the modules where we want the shaded classes to appear in the
|
|
associated jar. -->
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-http</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-continuation</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-servlet</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-servlets</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-proxy</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-client</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-security</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-plus</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-webapp</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<version>${guava.version}</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.jpmml</groupId>
|
|
<artifactId>pmml-model</artifactId>
|
|
<version>1.4.8</version>
|
|
<scope>provided</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.jpmml</groupId>
|
|
<artifactId>pmml-agent</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- End of shaded deps -->
|
|
|
|
<!-- Provide a JAXB impl; no longer auto available in Java 9+ in the JDK -->
|
|
<dependency>
|
|
<groupId>org.glassfish.jaxb</groupId>
|
|
<artifactId>jaxb-runtime</artifactId>
|
|
<version>2.3.2</version>
|
|
<scope>compile</scope>
|
|
<exclusions>
|
|
<!-- for now, we only write XML in PMML export, and these can be excluded -->
|
|
<exclusion>
|
|
<groupId>com.sun.xml.fastinfoset</groupId>
|
|
<artifactId>FastInfoset</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.glassfish.jaxb</groupId>
|
|
<artifactId>txw2</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jvnet.staxex</groupId>
|
|
<artifactId>stax-ex</artifactId>
|
|
</exclusion>
|
|
<!--
|
|
SPARK-27611: Exclude redundant javax.activation implementation, which
|
|
conflicts with the existing javax.activation:activation:1.1.1 dependency.
|
|
-->
|
|
<exclusion>
|
|
<groupId>jakarta.activation</groupId>
|
|
<artifactId>jakarta.activation-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
<version>${commons-lang3.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-text</artifactId>
|
|
<version>1.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-lang</groupId>
|
|
<artifactId>commons-lang</artifactId>
|
|
<version>${commons-lang2.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-io</groupId>
|
|
<artifactId>commons-io</artifactId>
|
|
<version>${commons-io.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
<version>${commons-codec.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-math3</artifactId>
|
|
<version>${commons.math3.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-collections</groupId>
|
|
<artifactId>commons-collections</artifactId>
|
|
<version>${commons.collections.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-beanutils</groupId>
|
|
<artifactId>commons-beanutils</artifactId>
|
|
<version>1.9.4</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
<!-- Hive uses commons-logging 1.1.3 from 0.13 to 1.2 -->
|
|
<version>1.1.3</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ivy</groupId>
|
|
<artifactId>ivy</artifactId>
|
|
<version>${ivy.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
<version>${jsr305.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-httpclient</groupId>
|
|
<artifactId>commons-httpclient</artifactId>
|
|
<version>${httpclient.classic.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
<version>${commons.httpclient.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpmime</artifactId>
|
|
<version>${commons.httpclient.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpcore</artifactId>
|
|
<version>${commons.httpcore.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${leveldbjni.group}</groupId>
|
|
<artifactId>leveldbjni-all</artifactId>
|
|
<version>1.8</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.seleniumhq.selenium</groupId>
|
|
<artifactId>selenium-java</artifactId>
|
|
<version>${selenium.version}</version>
|
|
<scope>test</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>net.bytebuddy</groupId>
|
|
<artifactId>byte-buddy</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.seleniumhq.selenium</groupId>
|
|
<artifactId>htmlunit-driver</artifactId>
|
|
<version>${htmlunit.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Update htmlunit dependency that selenium uses for better JS support -->
|
|
<dependency>
|
|
<groupId>net.sourceforge.htmlunit</groupId>
|
|
<artifactId>htmlunit</artifactId>
|
|
<version>${htmlunit.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.sourceforge.htmlunit</groupId>
|
|
<artifactId>htmlunit-core-js</artifactId>
|
|
<version>${htmlunit.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Added for selenium only, and should match its dependent version: -->
|
|
<dependency>
|
|
<groupId>xml-apis</groupId>
|
|
<artifactId>xml-apis</artifactId>
|
|
<version>1.4.01</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jul-to-slf4j</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>jcl-over-slf4j</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<!-- runtime scope is appropriate, but causes SBT build problems -->
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
<version>${log4j.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.ning</groupId>
|
|
<artifactId>compress-lzf</artifactId>
|
|
<version>1.0.3</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.xerial.snappy</groupId>
|
|
<artifactId>snappy-java</artifactId>
|
|
<version>${snappy.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.lz4</groupId>
|
|
<artifactId>lz4-java</artifactId>
|
|
<version>1.7.1</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.github.luben</groupId>
|
|
<artifactId>zstd-jni</artifactId>
|
|
<version>1.4.5-6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.clearspring.analytics</groupId>
|
|
<artifactId>stream</artifactId>
|
|
<version>2.9.6</version>
|
|
<exclusions>
|
|
<!-- Only HyperLogLogPlus is used, which doesn't depend on fastutil -->
|
|
<exclusion>
|
|
<groupId>it.unimi.dsi</groupId>
|
|
<artifactId>fastutil</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- In theory we need not directly depend on protobuf since Spark does not directly
|
|
use it. However, when building with Hadoop/YARN 2.2 Maven doesn't correctly bump
|
|
the protobuf version up from the one Mesos gives. For now we include this variable
|
|
to explicitly bump the version when building with YARN. It would be nice to figure
|
|
out why Maven can't resolve this correctly (like SBT does). -->
|
|
<dependency>
|
|
<groupId>com.google.protobuf</groupId>
|
|
<artifactId>protobuf-java</artifactId>
|
|
<version>${protobuf.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.roaringbitmap</groupId>
|
|
<artifactId>RoaringBitmap</artifactId>
|
|
<version>0.9.0</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-net</groupId>
|
|
<artifactId>commons-net</artifactId>
|
|
<version>3.1</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-all</artifactId>
|
|
<version>4.1.51.Final</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.derby</groupId>
|
|
<artifactId>derby</artifactId>
|
|
<version>${derby.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-core</artifactId>
|
|
<version>${codahale.metrics.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-jvm</artifactId>
|
|
<version>${codahale.metrics.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-json</artifactId>
|
|
<version>${codahale.metrics.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-graphite</artifactId>
|
|
<version>${codahale.metrics.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>io.dropwizard.metrics</groupId>
|
|
<artifactId>metrics-jmx</artifactId>
|
|
<version>${codahale.metrics.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
</dependency>
|
|
<!-- Guava is excluded because of SPARK-6149. The Guava version referenced in this module is
|
|
15.0, which causes runtime incompatibility issues. -->
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.module</groupId>
|
|
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.module</groupId>
|
|
<artifactId>jackson-module-jaxb-annotations</artifactId>
|
|
<version>${fasterxml.jackson.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.core</groupId>
|
|
<artifactId>jersey-server</artifactId>
|
|
<version>${jersey.version}</version>
|
|
<!-- SPARK-28765 Unused JDK11-specific dependency -->
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>jakarta.xml.bind</groupId>
|
|
<artifactId>jakarta.xml.bind-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.core</groupId>
|
|
<artifactId>jersey-common</artifactId>
|
|
<version>${jersey.version}</version>
|
|
<!-- SPARK-28765 Unused JDK11-specific dependency -->
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.sun.activation</groupId>
|
|
<artifactId>jakarta.activation</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.core</groupId>
|
|
<artifactId>jersey-client</artifactId>
|
|
<version>${jersey.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.containers</groupId>
|
|
<artifactId>jersey-container-servlet</artifactId>
|
|
<version>${jersey.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.containers</groupId>
|
|
<artifactId>jersey-container-servlet-core</artifactId>
|
|
<version>${jersey.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.inject</groupId>
|
|
<artifactId>jersey-hk2</artifactId>
|
|
<version>${jersey.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey.test-framework.providers</groupId>
|
|
<artifactId>jersey-test-framework-provider-simple</artifactId>
|
|
<version>${jersey.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.glassfish.jersey</groupId>
|
|
<artifactId>jersey-client</artifactId>
|
|
<version>${jersey.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>javax.ws.rs</groupId>
|
|
<artifactId>javax.ws.rs-api</artifactId>
|
|
<version>2.0.1</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>javax.xml.bind</groupId>
|
|
<artifactId>jaxb-api</artifactId>
|
|
<version>2.2.11</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalanlp</groupId>
|
|
<artifactId>breeze_${scala.binary.version}</artifactId>
|
|
<version>1.0</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-math3</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.json4s</groupId>
|
|
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
|
|
<version>3.7.0-M5</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang.modules</groupId>
|
|
<artifactId>scala-xml_${scala.binary.version}</artifactId>
|
|
<version>1.2.0</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-compiler</artifactId>
|
|
<version>${scala.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-reflect</artifactId>
|
|
<version>${scala.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
<version>${scala.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-actors</artifactId>
|
|
<version>${scala.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scala-lang.modules</groupId>
|
|
<artifactId>scala-parser-combinators_${scala.binary.version}</artifactId>
|
|
<version>1.1.2</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>jline</groupId>
|
|
<artifactId>jline</artifactId>
|
|
<version>2.14.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
|
<version>3.2.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-scalacheck_${scala.binary.version}</artifactId>
|
|
<version>3.1.0.0-RC2</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-mockito_${scala.binary.version}</artifactId>
|
|
<version>1.0.0-SNAP5</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalatestplus</groupId>
|
|
<artifactId>scalatestplus-selenium_${scala.binary.version}</artifactId>
|
|
<version>1.0.0-SNAP5</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-core</artifactId>
|
|
<version>3.1.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.jmock</groupId>
|
|
<artifactId>jmock-junit4</artifactId>
|
|
<scope>test</scope>
|
|
<version>2.8.4</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.scalacheck</groupId>
|
|
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
|
<version>1.14.2</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<version>4.12</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.hamcrest</groupId>
|
|
<artifactId>hamcrest-core</artifactId>
|
|
<version>1.3</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.hamcrest</groupId>
|
|
<artifactId>hamcrest-library</artifactId>
|
|
<version>1.3</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.novocode</groupId>
|
|
<artifactId>junit-interface</artifactId>
|
|
<version>0.11</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.spotify</groupId>
|
|
<artifactId>docker-client</artifactId>
|
|
<version>8.14.1</version>
|
|
<scope>test</scope>
|
|
<classifier>shaded</classifier>
|
|
<exclusions>
|
|
<exclusion>
|
|
<artifactId>guava</artifactId>
|
|
<groupId>com.google.guava</groupId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>mysql</groupId>
|
|
<artifactId>mysql-connector-java</artifactId>
|
|
<version>5.1.38</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.mariadb.jdbc</groupId>
|
|
<artifactId>mariadb-java-client</artifactId>
|
|
<version>2.5.4</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.postgresql</groupId>
|
|
<artifactId>postgresql</artifactId>
|
|
<version>42.2.6</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.ibm.db2</groupId>
|
|
<artifactId>jcc</artifactId>
|
|
<version>11.5.0.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.microsoft.sqlserver</groupId>
|
|
<artifactId>mssql-jdbc</artifactId>
|
|
<version>8.2.2.jre8</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.oracle.database.jdbc</groupId>
|
|
<artifactId>ojdbc8</artifactId>
|
|
<version>19.6.0.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
<version>${curator.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>jline</groupId>
|
|
<artifactId>jline</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-client</artifactId>
|
|
<version>${curator.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
<version>${curator.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-test</artifactId>
|
|
<version>${curator.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- Hadoop 3.x dependencies -->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-api</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-runtime</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client-minicluster</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<!-- End of Hadoop 3.x dependencies -->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.fusesource.leveldbjni</groupId>
|
|
<artifactId>leveldbjni-all</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<!-- BeanUtils >= 1.9.0 no longer splits out -core; exclude it -->
|
|
<groupId>commons-beanutils</groupId>
|
|
<artifactId>commons-beanutils-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-all</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>servlet-api-2.5</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>net.java.dev.jets3t</groupId>
|
|
<artifactId>jets3t</artifactId>
|
|
</exclusion>
|
|
<!-- Hadoop-3.2 -->
|
|
<exclusion>
|
|
<groupId>javax.ws.rs</groupId>
|
|
<artifactId>jsr311-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-webapp</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-minikdc</artifactId>
|
|
<version>${hadoop.version}</version>
|
|
<scope>test</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.directory.api</groupId>
|
|
<artifactId>api-ldap-schema-data</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- Managed up to match Hadoop in HADOOP-16530 -->
|
|
<dependency>
|
|
<groupId>xerces</groupId>
|
|
<artifactId>xercesImpl</artifactId>
|
|
<version>2.12.0</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
<version>${avro.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-ipc</artifactId>
|
|
<version>${avro.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>jetty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.velocity</groupId>
|
|
<artifactId>velocity</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- avro-mapred for some reason depends on avro-ipc's test jar, so undo that. -->
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-ipc</artifactId>
|
|
<classifier>tests</classifier>
|
|
<version>${avro.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-mapred</artifactId>
|
|
<version>${avro.version}</version>
|
|
<classifier>${avro.mapred.classifier}</classifier>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>jetty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>jetty-util</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.velocity</groupId>
|
|
<artifactId>velocity</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- See SPARK-23654 for info on this dependency;
|
|
It is used to keep javax.activation at v1.1.1 after dropping
|
|
jets3t as a dependency.
|
|
-->
|
|
<dependency>
|
|
<groupId>javax.activation</groupId>
|
|
<artifactId>activation</artifactId>
|
|
<version>1.1.1</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-api</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>jdk.tools</groupId>
|
|
<artifactId>jdk.tools</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-common</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-tests</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.fusesource.leveldbjni</groupId>
|
|
<artifactId>leveldbjni-all</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!--
|
|
Hack to exclude org.apache.hadoop:hadoop-yarn-server-resourcemanager:jar:tests.
|
|
For some reasons, SBT starts to pull the dependencies of 'hadoop-yarn-server-tests' above
|
|
with 'tests' classifier after upgrading SBT 1.3 (SPARK-21708). Otherwise, some tests might
|
|
fail, see also SPARK-33104.
|
|
-->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.fusesource.leveldbjni</groupId>
|
|
<artifactId>leveldbjni-all</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<!-- Hadoop-3.2 -->
|
|
<exclusion>
|
|
<groupId>com.zaxxer</groupId>
|
|
<artifactId>HikariCP-java7</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.microsoft.sqlserver</groupId>
|
|
<artifactId>mssql-jdbc</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-client</artifactId>
|
|
<version>${yarn.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.jersey-test-framework</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.sun.jersey.contribs</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
<version>${zookeeper.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.jboss.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>jline</groupId>
|
|
<artifactId>jline</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.github.spotbugs</groupId>
|
|
<artifactId>spotbugs-annotations</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-core-asl</artifactId>
|
|
<version>${codehaus.jackson.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-mapper-asl</artifactId>
|
|
<version>${codehaus.jackson.version}</version>
|
|
<scope>${hadoop.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-xc</artifactId>
|
|
<version>${codehaus.jackson.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>jackson-jaxrs</artifactId>
|
|
<version>${codehaus.jackson.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-beeline</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-exec</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-jdbc</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service-rpc</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-cli</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-exec</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-jdbc</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.ant</groupId>
|
|
<artifactId>ant</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-auth</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<!-- Begin of Hive 2.3 exclusion -->
|
|
<!--
|
|
ORC is needed, but the version should be consistent with the `sql/core` ORC data source.
|
|
Looks like this is safe, please see the major changes from ORC 1.3.3 to 1.5.4:
|
|
HIVE-17631 and HIVE-19465
|
|
-->
|
|
<exclusion>
|
|
<groupId>org.apache.orc</groupId>
|
|
<artifactId>orc-core</artifactId>
|
|
</exclusion>
|
|
<!-- jetty-all conflict with jetty 9.4.12.v20180830 -->
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty.aggregate</groupId>
|
|
<artifactId>jetty-all</artifactId>
|
|
</exclusion>
|
|
<!-- org.apache.logging.log4j:* conflict with log4j 1.2.17 -->
|
|
<exclusion>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<!-- Hive includes javax.servlet to fix the Hive on Spark test failure; see HIVE-12783 -->
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty.orbit</groupId>
|
|
<artifactId>javax.servlet</artifactId>
|
|
</exclusion>
|
|
<!-- hive-storage-api is needed and must be explicitly included later -->
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-storage-api</artifactId>
|
|
</exclusion>
|
|
<!-- End of Hive 2.3 exclusion -->
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-exec</artifactId>
|
|
<classifier>${hive.classifier}</classifier>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
|
|
<!-- pull this in when needed; the explicit definition culls the surplis-->
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-ant</artifactId>
|
|
</exclusion>
|
|
<!-- break the loop -->
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>spark-client</artifactId>
|
|
</exclusion>
|
|
|
|
<!-- excluded dependencies & transitive.
|
|
Some may be needed to be explicitly included-->
|
|
<exclusion>
|
|
<groupId>ant</groupId>
|
|
<artifactId>ant</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.ant</groupId>
|
|
<artifactId>ant</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.esotericsoftware.kryo</groupId>
|
|
<artifactId>kryo</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-httpclient</groupId>
|
|
<artifactId>commons-httpclient</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-mapred</artifactId>
|
|
</exclusion>
|
|
<!-- Do not need Calcite because we disabled hive.cbo.enable -->
|
|
<exclusion>
|
|
<groupId>org.apache.calcite</groupId>
|
|
<artifactId>calcite-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.calcite</groupId>
|
|
<artifactId>calcite-avatica</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>apache-curator</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-client</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.groovy</groupId>
|
|
<artifactId>groovy-all</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>jline</groupId>
|
|
<artifactId>jline</artifactId>
|
|
</exclusion>
|
|
<!-- Cat X license now; see SPARK-18262 -->
|
|
<exclusion>
|
|
<groupId>org.json</groupId>
|
|
<artifactId>json</artifactId>
|
|
</exclusion>
|
|
<!-- Begin of Hive 2.3 exclusion -->
|
|
<!-- Do not need Tez -->
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-llap-tez</artifactId>
|
|
</exclusion>
|
|
<!-- Do not need Calcite, see SPARK-27054 -->
|
|
<exclusion>
|
|
<groupId>org.apache.calcite</groupId>
|
|
<artifactId>calcite-druid</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.calcite.avatica</groupId>
|
|
<artifactId>avatica</artifactId>
|
|
</exclusion>
|
|
<!-- org.apache.logging.log4j:* conflict with log4j 1.2.17 -->
|
|
<exclusion>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<!-- End of Hive 2.3 exclusion -->
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-jdbc</artifactId>
|
|
<version>${hive.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service-rpc</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.groovy</groupId>
|
|
<artifactId>groovy-all</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<!-- Begin of Hive 2.3 exclusion -->
|
|
<!-- Hive removes the HBase Metastore; see HIVE-17234 -->
|
|
<exclusion>
|
|
<groupId>org.apache.hbase</groupId>
|
|
<artifactId>hbase-client</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>co.cask.tephra</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<!-- End of Hive 2.3 exclusion -->
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.google.code.findbugs</groupId>
|
|
<artifactId>jsr305</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.groovy</groupId>
|
|
<artifactId>groovy-all</artifactId>
|
|
</exclusion>
|
|
<!-- Begin of Hive 2.3 exclusion -->
|
|
<exclusion>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service-rpc</artifactId>
|
|
</exclusion>
|
|
<!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
|
|
<exclusion>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-hadoop-bundle</artifactId>
|
|
</exclusion>
|
|
<!-- Do not need Jasper, see HIVE-19799 -->
|
|
<exclusion>
|
|
<groupId>tomcat</groupId>
|
|
<artifactId>jasper-compiler</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>tomcat</groupId>
|
|
<artifactId>jasper-runtime</artifactId>
|
|
</exclusion>
|
|
<!-- End of Hive 2.3 exclusion -->
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-service-rpc</artifactId>
|
|
<version>${hive.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>*</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.sf.jpam</groupId>
|
|
<artifactId>jpam</artifactId>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<version>${jpam.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>servlet-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- hive shims pulls in hive 0.23 and a transitive dependency of the Hadoop version
|
|
Hive was built against. This dependency cuts out the YARN/hadoop dependency, which
|
|
is needed by Hive to submit work to a YARN cluster.-->
|
|
<dependency>
|
|
<groupId>${hive.group}</groupId>
|
|
<artifactId>hive-shims</artifactId>
|
|
<version>${hive.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-log4j12</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.groovy</groupId>
|
|
<artifactId>groovy-all</artifactId>
|
|
</exclusion>
|
|
<!-- Begin of Hive 2.3 exclusion -->
|
|
<!-- Exclude log4j-slf4j-impl, otherwise throw NCDFE when starting spark-shell -->
|
|
<exclusion>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-slf4j-impl</artifactId>
|
|
</exclusion>
|
|
<!-- End of Hive 2.3 exclusion -->
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- hive-llap-common is needed when registering UDFs in Hive 2.3.
|
|
We add it here, otherwise -Phive-provided won't work. -->
|
|
<dependency>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-llap-common</artifactId>
|
|
<version>${hive23.version}</version>
|
|
<scope>${hive.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<!-- hive-llap-client is needed when run MapReduce test in Hive 2.3. -->
|
|
<dependency>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-llap-client</artifactId>
|
|
<version>${hive23.version}</version>
|
|
<scope>test</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-serde</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-llap-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>apache-curator</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.zookeeper</groupId>
|
|
<artifactId>zookeeper</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.orc</groupId>
|
|
<artifactId>orc-core</artifactId>
|
|
<version>${orc.version}</version>
|
|
<scope>${orc.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.xml.bind</groupId>
|
|
<artifactId>jaxb-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-storage-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.orc</groupId>
|
|
<artifactId>orc-mapreduce</artifactId>
|
|
<version>${orc.version}</version>
|
|
<scope>${orc.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.orc</groupId>
|
|
<artifactId>orc-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-storage-api</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId> com.esotericsoftware</groupId>
|
|
<artifactId>kryo-shaded</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-column</artifactId>
|
|
<version>${parquet.version}</version>
|
|
<scope>${parquet.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-hadoop</artifactId>
|
|
<version>${parquet.version}</version>
|
|
<scope>${parquet.deps.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>commons-pool</groupId>
|
|
<artifactId>commons-pool</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-avro</artifactId>
|
|
<version>${parquet.version}</version>
|
|
<scope>${parquet.test.deps.scope}</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.janino</groupId>
|
|
<artifactId>janino</artifactId>
|
|
<version>${janino.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.codehaus.janino</groupId>
|
|
<artifactId>commons-compiler</artifactId>
|
|
<version>${janino.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>joda-time</groupId>
|
|
<artifactId>joda-time</artifactId>
|
|
<version>${joda.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.jodd</groupId>
|
|
<artifactId>jodd-core</artifactId>
|
|
<version>${jodd.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.datanucleus</groupId>
|
|
<artifactId>datanucleus-core</artifactId>
|
|
<version>${datanucleus-core.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libthrift</artifactId>
|
|
<version>${libthrift.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.thrift</groupId>
|
|
<artifactId>libfb303</artifactId>
|
|
<version>0.9.3</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.antlr</groupId>
|
|
<artifactId>antlr4-runtime</artifactId>
|
|
<version>${antlr4.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-crypto</artifactId>
|
|
<version>${commons-crypto.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>net.java.dev.jna</groupId>
|
|
<artifactId>jna</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.thoughtworks.paranamer</groupId>
|
|
<artifactId>paranamer</artifactId>
|
|
<version>${paranamer.version}</version>
|
|
<scope>runtime</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.arrow</groupId>
|
|
<artifactId>arrow-vector</artifactId>
|
|
<version>${arrow.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-core</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-common</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.arrow</groupId>
|
|
<artifactId>arrow-memory-netty</artifactId>
|
|
<version>${arrow.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-buffer</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>io.netty</groupId>
|
|
<artifactId>netty-common</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.univocity</groupId>
|
|
<artifactId>univocity-parsers</artifactId>
|
|
<version>2.9.0</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hive</groupId>
|
|
<artifactId>hive-storage-api</artifactId>
|
|
<version>${hive.storage.version}</version>
|
|
<scope>${hive.storage.scope}</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>commons-lang</groupId>
|
|
<artifactId>commons-lang</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
</dependencies>
|
|
</dependencyManagement>
|
|
|
|
<build>
|
|
<pluginManagement>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-enforcer-plugin</artifactId>
|
|
<version>3.0.0-M2</version>
|
|
<executions>
|
|
<execution>
|
|
<id>enforce-versions</id>
|
|
<goals>
|
|
<goal>enforce</goal>
|
|
</goals>
|
|
<configuration>
|
|
<rules>
|
|
<requireMavenVersion>
|
|
<version>${maven.version}</version>
|
|
</requireMavenVersion>
|
|
<requireJavaVersion>
|
|
<version>${java.version}</version>
|
|
</requireJavaVersion>
|
|
<bannedDependencies>
|
|
<excludes>
|
|
<exclude>org.jboss.netty</exclude>
|
|
<exclude>org.codehaus.groovy</exclude>
|
|
<exclude>*:*_2.11</exclude>
|
|
<exclude>*:*_2.10</exclude>
|
|
</excludes>
|
|
<searchTransitive>true</searchTransitive>
|
|
</bannedDependencies>
|
|
</rules>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
<version>3.2.0</version>
|
|
<executions>
|
|
<execution>
|
|
<id>module-timestamp-property</id>
|
|
<phase>validate</phase>
|
|
<goals>
|
|
<goal>timestamp-property</goal>
|
|
</goals>
|
|
<configuration>
|
|
<name>module.build.timestamp</name>
|
|
<pattern>${maven.build.timestamp.format}</pattern>
|
|
<timeSource>current</timeSource>
|
|
<timeZone>America/Los_Angeles</timeZone>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>local-timestamp-property</id>
|
|
<phase>validate</phase>
|
|
<goals>
|
|
<goal>timestamp-property</goal>
|
|
</goals>
|
|
<configuration>
|
|
<name>local.build.timestamp</name>
|
|
<pattern>${maven.build.timestamp.format}</pattern>
|
|
<timeSource>build</timeSource>
|
|
<timeZone>America/Los_Angeles</timeZone>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>net.alchim31.maven</groupId>
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
<version>4.3.0</version>
|
|
<executions>
|
|
<execution>
|
|
<id>eclipse-add-source</id>
|
|
<goals>
|
|
<goal>add-source</goal>
|
|
</goals>
|
|
</execution>
|
|
<execution>
|
|
<id>scala-compile-first</id>
|
|
<goals>
|
|
<goal>compile</goal>
|
|
</goals>
|
|
</execution>
|
|
<execution>
|
|
<id>scala-test-compile-first</id>
|
|
<goals>
|
|
<goal>testCompile</goal>
|
|
</goals>
|
|
</execution>
|
|
<execution>
|
|
<id>attach-scaladocs</id>
|
|
<phase>verify</phase>
|
|
<goals>
|
|
<goal>doc-jar</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<scalaVersion>${scala.version}</scalaVersion>
|
|
<checkMultipleScalaVersions>true</checkMultipleScalaVersions>
|
|
<failOnMultipleScalaVersions>true</failOnMultipleScalaVersions>
|
|
<recompileMode>incremental</recompileMode>
|
|
<useZincServer>true</useZincServer>
|
|
<args>
|
|
<arg>-unchecked</arg>
|
|
<arg>-deprecation</arg>
|
|
<arg>-feature</arg>
|
|
<arg>-explaintypes</arg>
|
|
<arg>-target:jvm-1.8</arg>
|
|
</args>
|
|
<jvmArgs>
|
|
<jvmArg>-Xms1024m</jvmArg>
|
|
<jvmArg>-Xmx1024m</jvmArg>
|
|
<jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
|
|
</jvmArgs>
|
|
<javacArgs>
|
|
<javacArg>-source</javacArg>
|
|
<javacArg>${java.version}</javacArg>
|
|
<javacArg>-target</javacArg>
|
|
<javacArg>${java.version}</javacArg>
|
|
<javacArg>-Xlint:all,-serial,-path,-try</javacArg>
|
|
</javacArgs>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
<version>3.8.1</version>
|
|
<configuration>
|
|
<source>${java.version}</source>
|
|
<target>${java.version}</target>
|
|
<skipMain>true</skipMain> <!-- skip compile -->
|
|
<skip>true</skip> <!-- skip testCompile -->
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.antlr</groupId>
|
|
<artifactId>antlr4-maven-plugin</artifactId>
|
|
<version>${antlr4.version}</version>
|
|
</plugin>
|
|
<!-- Surefire runs all Java tests -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<version>3.0.0-M3</version>
|
|
<!-- Note config is repeated in scalatest config -->
|
|
<configuration>
|
|
<includes>
|
|
<include>**/Test*.java</include>
|
|
<include>**/*Test.java</include>
|
|
<include>**/*TestCase.java</include>
|
|
<include>**/*Suite.java</include>
|
|
</includes>
|
|
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
|
|
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
|
|
<environmentVariables>
|
|
<!--
|
|
Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
|
|
launched by the tests have access to the correct test-time classpath.
|
|
-->
|
|
<SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
|
|
<SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
|
|
<SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
|
|
<SPARK_TESTING>1</SPARK_TESTING>
|
|
<JAVA_HOME>${test.java.home}</JAVA_HOME>
|
|
</environmentVariables>
|
|
<systemProperties>
|
|
<log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
|
|
<derby.system.durability>test</derby.system.durability>
|
|
<java.awt.headless>true</java.awt.headless>
|
|
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
|
|
<spark.test.home>${spark.test.home}</spark.test.home>
|
|
<spark.testing>1</spark.testing>
|
|
<spark.master.rest.enabled>false</spark.master.rest.enabled>
|
|
<spark.ui.enabled>false</spark.ui.enabled>
|
|
<spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
|
|
<spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
|
|
<spark.memory.debugFill>true</spark.memory.debugFill>
|
|
<!-- Needed by sql/hive tests. -->
|
|
<test.src.tables>src</test.src.tables>
|
|
</systemProperties>
|
|
<failIfNoTests>false</failIfNoTests>
|
|
<excludedGroups>${test.exclude.tags}</excludedGroups>
|
|
<groups>${test.include.tags}</groups>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>test</id>
|
|
<goals>
|
|
<goal>test</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<!-- Scalatest runs all Scala tests -->
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
<version>${scalatest-maven-plugin.version}</version>
|
|
<!-- Note config is repeated in surefire config -->
|
|
<configuration>
|
|
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
|
|
<junitxml>.</junitxml>
|
|
<filereports>SparkTestSuite.txt</filereports>
|
|
<argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
|
|
<stderr/>
|
|
<environmentVariables>
|
|
<!--
|
|
Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
|
|
launched by the tests have access to the correct test-time classpath.
|
|
-->
|
|
<SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
|
|
<SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
|
|
<SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
|
|
<SPARK_TESTING>1</SPARK_TESTING>
|
|
<JAVA_HOME>${test.java.home}</JAVA_HOME>
|
|
</environmentVariables>
|
|
<systemProperties>
|
|
<log4j.configuration>file:src/test/resources/log4j.properties</log4j.configuration>
|
|
<derby.system.durability>test</derby.system.durability>
|
|
<java.awt.headless>true</java.awt.headless>
|
|
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
|
|
<spark.test.home>${spark.test.home}</spark.test.home>
|
|
<spark.testing>1</spark.testing>
|
|
<spark.ui.enabled>false</spark.ui.enabled>
|
|
<spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
|
|
<spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
|
|
<spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
|
|
<!-- Needed by sql/hive tests. -->
|
|
<test.src.tables>__not_used__</test.src.tables>
|
|
</systemProperties>
|
|
<tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
|
|
<tagsToInclude>${test.include.tags}</tagsToInclude>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>test</id>
|
|
<goals>
|
|
<goal>test</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<version>3.1.2</version>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<version>${maven-antrun.version}</version>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-source-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<configuration>
|
|
<attach>true</attach>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>create-source-jar</id>
|
|
<goals>
|
|
<goal>jar-no-fork</goal>
|
|
<goal>test-jar-no-fork</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-clean-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<configuration>
|
|
<filesets>
|
|
<fileset>
|
|
<directory>work</directory>
|
|
</fileset>
|
|
<fileset>
|
|
<directory>checkpoint</directory>
|
|
</fileset>
|
|
<fileset>
|
|
<directory>lib_managed</directory>
|
|
</fileset>
|
|
<fileset>
|
|
<directory>metastore_db</directory>
|
|
</fileset>
|
|
<fileset>
|
|
<directory>spark-warehouse</directory>
|
|
</fileset>
|
|
</filesets>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-javadoc-plugin</artifactId>
|
|
<version>3.1.1</version>
|
|
<configuration>
|
|
<additionalJOptions>
|
|
<additionalJOption>-Xdoclint:all</additionalJOption>
|
|
<additionalJOption>-Xdoclint:-missing</additionalJOption>
|
|
</additionalJOptions>
|
|
<tags>
|
|
<tag>
|
|
<name>example</name>
|
|
<placement>a</placement>
|
|
<head>Example:</head>
|
|
</tag>
|
|
<tag>
|
|
<name>note</name>
|
|
<placement>a</placement>
|
|
<head>Note:</head>
|
|
</tag>
|
|
<tag>
|
|
<name>group</name>
|
|
<placement>X</placement>
|
|
</tag>
|
|
<tag>
|
|
<name>tparam</name>
|
|
<placement>X</placement>
|
|
</tag>
|
|
<tag>
|
|
<name>constructor</name>
|
|
<placement>X</placement>
|
|
</tag>
|
|
<tag>
|
|
<name>todo</name>
|
|
<placement>X</placement>
|
|
</tag>
|
|
<tag>
|
|
<name>groupname</name>
|
|
<placement>X</placement>
|
|
</tag>
|
|
</tags>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>exec-maven-plugin</artifactId>
|
|
<version>${exec-maven-plugin.version}</version>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-assembly-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<configuration>
|
|
<tarLongFileMode>posix</tarLongFileMode>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<version>3.2.1</version>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm</artifactId>
|
|
<version>7.2</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.ow2.asm</groupId>
|
|
<artifactId>asm-commons</artifactId>
|
|
<version>7.2</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-install-plugin</artifactId>
|
|
<version>3.0.0-M1</version>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-deploy-plugin</artifactId>
|
|
<version>3.0.0-M1</version>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<version>3.1.1</version>
|
|
<executions>
|
|
<execution>
|
|
<id>default-cli</id>
|
|
<goals>
|
|
<goal>build-classpath</goal>
|
|
</goals>
|
|
<configuration>
|
|
<!-- This includes dependencies with 'runtime' and 'compile' scopes;
|
|
see the docs for includeScope for more details -->
|
|
<includeScope>runtime</includeScope>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<!-- This plugin's configuration is used to store Eclipse m2e settings only. -->
|
|
<!-- It has no influence on the Maven build itself. -->
|
|
<plugin>
|
|
<groupId>org.eclipse.m2e</groupId>
|
|
<artifactId>lifecycle-mapping</artifactId>
|
|
<version>1.0.0</version>
|
|
<configuration>
|
|
<lifecycleMappingMetadata>
|
|
<pluginExecutions>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<versionRange>[2.8,)</versionRange>
|
|
<goals>
|
|
<goal>build-classpath</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<versionRange>3.1.2</versionRange>
|
|
<goals>
|
|
<goal>test-jar</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<versionRange>[${maven-antrun.version},)</versionRange>
|
|
<goals>
|
|
<goal>run</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
</pluginExecutions>
|
|
</lifecycleMappingMetadata>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
</pluginManagement>
|
|
|
|
<plugins>
|
|
<!-- This plugin dumps the test classpath into a file -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>generate-test-classpath</id>
|
|
<phase>test-compile</phase>
|
|
<goals>
|
|
<goal>build-classpath</goal>
|
|
</goals>
|
|
<configuration>
|
|
<includeScope>test</includeScope>
|
|
<outputProperty>test_classpath</outputProperty>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>copy-module-dependencies</id>
|
|
<phase>${build.copyDependenciesPhase}</phase>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
<configuration>
|
|
<includeScope>runtime</includeScope>
|
|
<outputDirectory>${jars.target.dir}</outputDirectory>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<!--
|
|
The shade plug-in is used here to create effective pom's (see SPARK-3812), and also
|
|
remove references from the shaded libraries from artifacts published by Spark.
|
|
-->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<configuration>
|
|
<shadedArtifactAttached>false</shadedArtifactAttached>
|
|
<createDependencyReducedPom>false</createDependencyReducedPom>
|
|
<artifactSet>
|
|
<includes>
|
|
<include>org.spark-project.spark:unused</include>
|
|
<include>org.eclipse.jetty:jetty-io</include>
|
|
<include>org.eclipse.jetty:jetty-http</include>
|
|
<include>org.eclipse.jetty:jetty-proxy</include>
|
|
<include>org.eclipse.jetty:jetty-client</include>
|
|
<include>org.eclipse.jetty:jetty-continuation</include>
|
|
<include>org.eclipse.jetty:jetty-servlet</include>
|
|
<include>org.eclipse.jetty:jetty-servlets</include>
|
|
<include>org.eclipse.jetty:jetty-plus</include>
|
|
<include>org.eclipse.jetty:jetty-security</include>
|
|
<include>org.eclipse.jetty:jetty-util</include>
|
|
<include>org.eclipse.jetty:jetty-server</include>
|
|
<include>com.google.guava:guava</include>
|
|
<include>org.jpmml:*</include>
|
|
</includes>
|
|
</artifactSet>
|
|
<relocations>
|
|
<relocation>
|
|
<pattern>org.eclipse.jetty</pattern>
|
|
<shadedPattern>${spark.shade.packageName}.jetty</shadedPattern>
|
|
<includes>
|
|
<include>org.eclipse.jetty.**</include>
|
|
</includes>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>com.google.common</pattern>
|
|
<shadedPattern>${spark.shade.packageName}.guava</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.dmg.pmml</pattern>
|
|
<shadedPattern>${spark.shade.packageName}.dmg.pmml</shadedPattern>
|
|
</relocation>
|
|
<relocation>
|
|
<pattern>org.jpmml</pattern>
|
|
<shadedPattern>${spark.shade.packageName}.jpmml</shadedPattern>
|
|
</relocation>
|
|
</relocations>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-enforcer-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>net.alchim31.maven</groupId>
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-source-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.scalastyle</groupId>
|
|
<artifactId>scalastyle-maven-plugin</artifactId>
|
|
<version>1.0.0</version>
|
|
<configuration>
|
|
<verbose>false</verbose>
|
|
<failOnViolation>true</failOnViolation>
|
|
<includeTestSourceDirectory>false</includeTestSourceDirectory>
|
|
<failOnWarning>false</failOnWarning>
|
|
<sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
|
|
<testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
|
|
<configLocation>scalastyle-config.xml</configLocation>
|
|
<outputFile>${basedir}/target/scalastyle-output.xml</outputFile>
|
|
<inputEncoding>${project.build.sourceEncoding}</inputEncoding>
|
|
<outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>check</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-checkstyle-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<configuration>
|
|
<failOnViolation>false</failOnViolation>
|
|
<includeTestSourceDirectory>true</includeTestSourceDirectory>
|
|
<sourceDirectories>
|
|
<directory>${basedir}/src/main/java</directory>
|
|
<directory>${basedir}/src/main/scala</directory>
|
|
</sourceDirectories>
|
|
<testSourceDirectories>
|
|
<directory>${basedir}/src/test/java</directory>
|
|
</testSourceDirectories>
|
|
<configLocation>dev/checkstyle.xml</configLocation>
|
|
<outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
|
|
<inputEncoding>${project.build.sourceEncoding}</inputEncoding>
|
|
<outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
|
|
</configuration>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>com.puppycrawl.tools</groupId>
|
|
<artifactId>checkstyle</artifactId>
|
|
<version>8.29</version>
|
|
</dependency>
|
|
</dependencies>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>check</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>create-tmp-dir</id>
|
|
<phase>generate-test-resources</phase>
|
|
<goals>
|
|
<goal>run</goal>
|
|
</goals>
|
|
<configuration>
|
|
<target>
|
|
<mkdir dir="${project.build.directory}/tmp" />
|
|
</target>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<!-- Enable surefire and scalatest in all children, in one place: -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
</plugin>
|
|
<!-- Build test-jar's for all projects, since some projects depend on tests from others -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>prepare-test-jar</id>
|
|
<phase>${build.testJarPhase}</phase>
|
|
<goals>
|
|
<goal>test-jar</goal>
|
|
</goals>
|
|
<configuration>
|
|
<excludes>
|
|
<exclude>log4j.properties</exclude>
|
|
</excludes>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.antipathy</groupId>
|
|
<artifactId>mvn-scalafmt_${scala.binary.version}</artifactId>
|
|
<version>1.0.3</version>
|
|
<configuration>
|
|
<parameters>${scalafmt.parameters}</parameters> <!-- (Optional) Additional command line arguments -->
|
|
<skip>${scalafmt.skip}</skip> <!-- (Optional) skip formatting -->
|
|
<skipSources>${scalafmt.skip}</skipSources>
|
|
<skipTestSources>${scalafmt.skip}</skipTestSources>
|
|
<configLocation>dev/.scalafmt.conf</configLocation> <!-- (Optional) config location -->
|
|
<onlyChangedFiles>true</onlyChangedFiles>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<phase>validate</phase>
|
|
<goals>
|
|
<goal>format</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<!--
|
|
Couple of dependencies are coming in bundle format (bundle is just a normal jar which
|
|
contains OSGi metadata in the manifest). If one don't use OSGi, then a bundle will work as
|
|
any other jar. Since maven doesn't have native bundle support it needs an external plugin
|
|
handle it. If the plugin is not added then the build can't resolve bundle dependencies.
|
|
-->
|
|
<plugin>
|
|
<groupId>org.apache.felix</groupId>
|
|
<artifactId>maven-bundle-plugin</artifactId>
|
|
<version>4.2.0</version>
|
|
<extensions>true</extensions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<profiles>
|
|
|
|
<!--
|
|
This profile is enabled automatically by the sbt build. It changes the scope for shaded
|
|
dependencies, since we don't shade it in the artifacts generated by the sbt build.
|
|
-->
|
|
<profile>
|
|
<id>sbt</id>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.jpmml</groupId>
|
|
<artifactId>pmml-model</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
</profile>
|
|
|
|
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
|
|
<profile>
|
|
<id>spark-ganglia-lgpl</id>
|
|
<modules>
|
|
<module>external/spark-ganglia-lgpl</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<!-- Kinesis integration is not included by default due to ASL-licensed code -->
|
|
<profile>
|
|
<id>kinesis-asl</id>
|
|
<modules>
|
|
<module>external/kinesis-asl</module>
|
|
<module>external/kinesis-asl-assembly</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>docker-integration-tests</id>
|
|
<modules>
|
|
<module>external/docker-integration-tests</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<!-- A series of build profiles where customizations for particular Hadoop releases can be made -->
|
|
|
|
<!-- Hadoop-a.b.c dependencies can be found at
|
|
http://hadoop.apache.org/docs/ra.b.c/hadoop-project-dist/hadoop-common/dependency-analysis.html
|
|
-->
|
|
|
|
<profile>
|
|
<id>hadoop-2.7</id>
|
|
<properties>
|
|
<hadoop.version>2.7.4</hadoop.version>
|
|
<curator.version>2.7.1</curator.version>
|
|
<commons-io.version>2.4</commons-io.version>
|
|
<hadoop-client-api.artifact>hadoop-client</hadoop-client-api.artifact>
|
|
<hadoop-client-runtime.artifact>hadoop-client</hadoop-client-runtime.artifact>
|
|
<hadoop-client-minicluster.artifact>hadoop-client</hadoop-client-minicluster.artifact>
|
|
</properties>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>hadoop-3.2</id>
|
|
<!-- Default hadoop profile. Uses global properties. -->
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>hive-2.3</id>
|
|
<!-- Default hive profile. Uses global properties. -->
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>yarn</id>
|
|
<modules>
|
|
<module>resource-managers/yarn</module>
|
|
<module>common/network-yarn</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>mesos</id>
|
|
<modules>
|
|
<module>resource-managers/mesos</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>kubernetes</id>
|
|
<modules>
|
|
<module>resource-managers/kubernetes/core</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<!-- generally not enabled for automated builds, but will run k8s tests -->
|
|
<profile>
|
|
<id>kubernetes-integration-tests</id>
|
|
<modules>
|
|
<module>resource-managers/kubernetes/integration-tests</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>hive-thriftserver</id>
|
|
<modules>
|
|
<module>sql/hive-thriftserver</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>hadoop-cloud</id>
|
|
<modules>
|
|
<module>hadoop-cloud</module>
|
|
</modules>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>test-java-home</id>
|
|
<activation>
|
|
<property><name>env.JAVA_HOME</name></property>
|
|
</activation>
|
|
<properties>
|
|
<test.java.home>${env.JAVA_HOME}</test.java.home>
|
|
</properties>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>scala-2.12</id>
|
|
<build>
|
|
<pluginManagement>
|
|
<plugins>
|
|
</plugins>
|
|
</pluginManagement>
|
|
</build>
|
|
</profile>
|
|
|
|
<profile>
|
|
<id>scala-2.13</id>
|
|
<properties>
|
|
<scala.version>2.13.3</scala.version>
|
|
<scala.binary.version>2.13</scala.binary.version>
|
|
</properties>
|
|
<dependencyManagement>
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.scala-lang.modules</groupId>
|
|
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
|
|
<version>0.2.0</version>
|
|
</dependency>
|
|
</dependencies>
|
|
</dependencyManagement>
|
|
</profile>
|
|
|
|
<!--
|
|
This is a profile to enable the use of the ASF snapshot and staging repositories
|
|
during a build. It is useful when testing against nightly or RC releases of dependencies.
|
|
It MUST NOT be used when building copies of Spark to use in production of for distribution,
|
|
-->
|
|
<profile>
|
|
<id>snapshots-and-staging</id>
|
|
<properties>
|
|
<!-- override point for ASF staging/snapshot repos -->
|
|
<asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
|
|
<asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
|
|
</properties>
|
|
|
|
<pluginRepositories>
|
|
<pluginRepository>
|
|
<id>ASF Staging</id>
|
|
<url>${asf.staging}</url>
|
|
</pluginRepository>
|
|
<pluginRepository>
|
|
<id>ASF Snapshots</id>
|
|
<url>${asf.snapshots}</url>
|
|
<snapshots>
|
|
<enabled>true</enabled>
|
|
</snapshots>
|
|
<releases>
|
|
<enabled>false</enabled>
|
|
</releases>
|
|
</pluginRepository>
|
|
|
|
</pluginRepositories>
|
|
<repositories>
|
|
<repository>
|
|
<id>ASF Staging</id>
|
|
<url>${asf.staging}</url>
|
|
</repository>
|
|
<repository>
|
|
<id>ASF Snapshots</id>
|
|
<url>${asf.snapshots}</url>
|
|
<snapshots>
|
|
<enabled>true</enabled>
|
|
</snapshots>
|
|
<releases>
|
|
<enabled>false</enabled>
|
|
</releases>
|
|
</repository>
|
|
</repositories>
|
|
</profile>
|
|
|
|
<!--
|
|
These empty profiles are available in some sub-modules. Declare them here so that
|
|
maven does not complain when they're provided on the command line for a sub-module
|
|
that does not have them.
|
|
-->
|
|
<profile>
|
|
<id>hadoop-provided</id>
|
|
</profile>
|
|
<profile>
|
|
<id>hive-provided</id>
|
|
</profile>
|
|
<profile>
|
|
<id>orc-provided</id>
|
|
</profile>
|
|
<profile>
|
|
<id>parquet-provided</id>
|
|
</profile>
|
|
<profile>
|
|
<id>sparkr</id>
|
|
</profile>
|
|
<!-- use org.openlabtesting.leveldbjni on aarch64 platform -->
|
|
<profile>
|
|
<id>aarch64</id>
|
|
<properties>
|
|
<leveldbjni.group>org.openlabtesting.leveldbjni</leveldbjni.group>
|
|
</properties>
|
|
</profile>
|
|
</profiles>
|
|
</project>
|