[SPARK-36670][FOLLOWUP][TEST] Remove brotli-codec dependency

### What changes were proposed in this pull request?

Remove `com.github.rdblue:brotli-codec:0.1.1` dependency.

### Why are the changes needed?

As Stephen Coy pointed out in the dev list, we should not have `com.github.rdblue:brotli-codec:0.1.1` dependency which is not available on Maven Central. This is to avoid possible artifact changes on `Jitpack.io`.
Also, the dependency is for tests only. I suggest that we remove it now to unblock the 3.2.0 release ASAP.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

GA tests.

Closes #34059 from gengliangwang/removeDeps.

Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Gengliang Wang 2021-09-21 10:57:20 -07:00 committed by Dongjoon Hyun
parent 33e463ccf9
commit ba5708d944
4 changed files with 4 additions and 21 deletions

View file

@ -300,12 +300,6 @@
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
<name>Jitpack.io repository</name>
<!-- needed for brotli-codec -->
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>

View file

@ -274,9 +274,7 @@ object SparkBuild extends PomBuild {
"gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
DefaultMavenRepository,
Resolver.mavenLocal,
Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns),
// needed for brotli-codec
"jitpack.io" at "https://jitpack.io"
Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
),
externalResolvers := resolvers.value,
otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,

View file

@ -184,12 +184,6 @@
<artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.rdblue</groupId>
<artifactId>brotli-codec</artifactId>
<version>0.1.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>

View file

@ -56,13 +56,10 @@ class ParquetCodecSuite extends FileSourceCodecSuite {
override def format: String = "parquet"
override val codecConfigName: String = SQLConf.PARQUET_COMPRESSION.key
// Exclude "lzo" because it is GPL-licenced so not included in Hadoop.
// Exclude "brotli" because the com.github.rdblue:brotli-codec dependency is not available
// on Maven Central.
override protected def availableCodecs: Seq[String] =
if (System.getProperty("os.arch") == "aarch64") {
// Exclude "brotli" due to PARQUET-1975.
Seq("none", "uncompressed", "snappy", "lz4", "gzip", "zstd")
} else {
Seq("none", "uncompressed", "snappy", "lz4", "gzip", "brotli", "zstd")
}
Seq("none", "uncompressed", "snappy", "lz4", "gzip", "zstd")
}
class OrcCodecSuite extends FileSourceCodecSuite {