ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-36732][SQL][BUILD] Upgrade ORC to 1.6.11

Browse source 
### What changes were proposed in this pull request?

This PR aims to upgrade Apache ORC to 1.6.11 to bring the latest bug fixes.

### Why are the changes needed?

Apache ORC 1.6.11 has the following fixes.
- https://issues.apache.org/jira/projects/ORC/versions/12350499

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CIs.

Closes #33971 from dongjoon-hyun/SPARK-36732.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c217797297)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Dongjoon Hyun 2021-09-15 23:36:26 -07:00
parent 2067661869
commit 63b8417794
4 changed files with 9 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
dev/deps/spark-deps-hadoop-2.7-hive-2.3
View file

@ -195,9 +195,9 @@ objenesis/2.6//objenesis-2.6.jar
okhttp/3.12.12//okhttp-3.12.12.jar
okio/1.14.0//okio-1.14.0.jar
opencsv/2.3//opencsv-2.3.jar
orc-core/1.6.10//orc-core-1.6.10.jar
orc-mapreduce/1.6.10//orc-mapreduce-1.6.10.jar
orc-shims/1.6.10//orc-shims-1.6.10.jar
orc-core/1.6.11//orc-core-1.6.11.jar
orc-mapreduce/1.6.11//orc-mapreduce-1.6.11.jar
orc-shims/1.6.11//orc-shims-1.6.11.jar
oro/2.0.8//oro-2.0.8.jar
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8//paranamer-2.8.jar

7
dev/deps/spark-deps-hadoop-3.2-hive-2.3
View file

@ -102,7 +102,6 @@ janino/3.0.16//janino-3.0.16.jar
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
javolution/5.5.1//javolution-5.5.1.jar
jaxb-api/2.2.11//jaxb-api-2.2.11.jar
jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
jdo-api/3.0.1//jdo-api-3.0.1.jar
@ -166,9 +165,9 @@ objenesis/2.6//objenesis-2.6.jar
okhttp/3.12.12//okhttp-3.12.12.jar
okio/1.14.0//okio-1.14.0.jar
opencsv/2.3//opencsv-2.3.jar
orc-core/1.6.10//orc-core-1.6.10.jar
orc-mapreduce/1.6.10//orc-mapreduce-1.6.10.jar
orc-shims/1.6.10//orc-shims-1.6.10.jar
orc-core/1.6.11//orc-core-1.6.11.jar
orc-mapreduce/1.6.11//orc-mapreduce-1.6.11.jar
orc-shims/1.6.11//orc-shims-1.6.11.jar
oro/2.0.8//oro-2.0.8.jar
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8//paranamer-2.8.jar

7
pom.xml
View file

@ -137,7 +137,7 @@
<!-- After 10.15.1.3, the minimum required version is JDK9 -->
<derby.version>10.14.2.0</derby.version>
<parquet.version>1.12.1</parquet.version>
<orc.version>1.6.10</orc.version>
<orc.version>1.6.11</orc.version>
<jetty.version>9.4.43.v20210629</jetty.version>
<jakartaservlet.version>4.0.3</jakartaservlet.version>
<chill.version>0.10.0</chill.version>
@ -2305,11 +2305,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>aircompressor</artifactId>
<version>0.21</version>
</dependency>
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-mapreduce</artifactId>

4
sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
View file

@ -695,9 +695,9 @@ class FileBasedDataSourceSuite extends QueryTest
test("SPARK-22790,SPARK-27668: spark.sql.sources.compressionFactor takes effect") {
Seq(1.0, 0.5).foreach { compressionFactor =>
withSQLConf(SQLConf.FILE_COMPRESSION_FACTOR.key -> compressionFactor.toString,
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "250") {
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "350") {
withTempPath { workDir =>
// the file size is 486 bytes
// the file size is 504 bytes
val workDirPath = workDir.getAbsolutePath
val data1 = Seq(100, 200, 300, 400).toDF("count")
data1.write.orc(workDirPath + "/data1")