ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-31560][SQL][TESTS] Add V1/V2 tests for TextSuite and WholeTextFileSuite

Browse source 
### What changes were proposed in this pull request?

 Add V1/V2 tests for TextSuite and WholeTextFileSuite

### Why are the changes needed?

This is missing part since #24207. We should have these tests for test coverage.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Unit tests.

Closes #28335 from gengliangwang/testV2Suite.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Gengliang Wang 2020-04-24 18:59:15 -07:00 committed by Dongjoon Hyun
parent 9faad07ce7
commit 16b961526d
No known key found for this signature in database
GPG key ID: EDA00CE834F0FC5C
2 changed files with 32 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
View file

@ -24,14 +24,14 @@ import java.nio.file.Files
import org.apache.hadoop.io.SequenceFile.CompressionType
import org.apache.hadoop.io.compress.GzipCodec
import org.apache.spark.TestUtils
import org.apache.spark.{SparkConf, TestUtils}
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{StringType, StructType}
import org.apache.spark.util.Utils
class TextSuite extends QueryTest with SharedSparkSession {
abstract class TextSuite extends QueryTest with SharedSparkSession {
import testImplicits._
test("reading text file") {
@ -234,3 +234,17 @@ class TextSuite extends QueryTest with SharedSparkSession {
assert(data.length == 4)
}
}
class TextV1Suite extends TextSuite {
override protected def sparkConf: SparkConf =
super
.sparkConf
.set(SQLConf.USE_V1_SOURCE_LIST, "text")
}
class TextV2Suite extends TextSuite {
override protected def sparkConf: SparkConf =
super
.sparkConf
.set(SQLConf.USE_V1_SOURCE_LIST, "")
}

17
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
View file

@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.datasources.text
import java.io.File
import org.apache.spark.SparkConf
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{StringType, StructType}
class WholeTextFileSuite extends QueryTest with SharedSparkSession {
abstract class WholeTextFileSuite extends QueryTest with SharedSparkSession {
// Hadoop's FileSystem caching does not use the Configuration as part of its cache key, which
// can cause Filesystem.get(Configuration) to return a cached instance created with a different
@ -103,3 +104,17 @@ class WholeTextFileSuite extends QueryTest with SharedSparkSession {
}
}
}
class WholeTextFileV1Suite extends WholeTextFileSuite {
override protected def sparkConf: SparkConf =
super
.sparkConf
.set(SQLConf.USE_V1_SOURCE_LIST, "text")
}
class WholeTextFileV2Suite extends WholeTextFileSuite {
override protected def sparkConf: SparkConf =
super
.sparkConf
.set(SQLConf.USE_V1_SOURCE_LIST, "")
}