[SPARK-31560][SQL][TESTS] Add V1/V2 tests for TextSuite and WholeTextFileSuite
### What changes were proposed in this pull request? Add V1/V2 tests for TextSuite and WholeTextFileSuite ### Why are the changes needed? This is missing part since #24207. We should have these tests for test coverage. ### Does this PR introduce any user-facing change? No ### How was this patch tested? Unit tests. Closes #28335 from gengliangwang/testV2Suite. Authored-by: Gengliang Wang <gengliang.wang@databricks.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
parent
9faad07ce7
commit
16b961526d
|
@ -24,14 +24,14 @@ import java.nio.file.Files
|
|||
import org.apache.hadoop.io.SequenceFile.CompressionType
|
||||
import org.apache.hadoop.io.compress.GzipCodec
|
||||
|
||||
import org.apache.spark.TestUtils
|
||||
import org.apache.spark.{SparkConf, TestUtils}
|
||||
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode}
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.test.SharedSparkSession
|
||||
import org.apache.spark.sql.types.{StringType, StructType}
|
||||
import org.apache.spark.util.Utils
|
||||
|
||||
class TextSuite extends QueryTest with SharedSparkSession {
|
||||
abstract class TextSuite extends QueryTest with SharedSparkSession {
|
||||
import testImplicits._
|
||||
|
||||
test("reading text file") {
|
||||
|
@ -234,3 +234,17 @@ class TextSuite extends QueryTest with SharedSparkSession {
|
|||
assert(data.length == 4)
|
||||
}
|
||||
}
|
||||
|
||||
class TextV1Suite extends TextSuite {
|
||||
override protected def sparkConf: SparkConf =
|
||||
super
|
||||
.sparkConf
|
||||
.set(SQLConf.USE_V1_SOURCE_LIST, "text")
|
||||
}
|
||||
|
||||
class TextV2Suite extends TextSuite {
|
||||
override protected def sparkConf: SparkConf =
|
||||
super
|
||||
.sparkConf
|
||||
.set(SQLConf.USE_V1_SOURCE_LIST, "")
|
||||
}
|
||||
|
|
|
@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.datasources.text
|
|||
|
||||
import java.io.File
|
||||
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.{QueryTest, Row}
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.test.SharedSparkSession
|
||||
import org.apache.spark.sql.types.{StringType, StructType}
|
||||
|
||||
class WholeTextFileSuite extends QueryTest with SharedSparkSession {
|
||||
abstract class WholeTextFileSuite extends QueryTest with SharedSparkSession {
|
||||
|
||||
// Hadoop's FileSystem caching does not use the Configuration as part of its cache key, which
|
||||
// can cause Filesystem.get(Configuration) to return a cached instance created with a different
|
||||
|
@ -103,3 +104,17 @@ class WholeTextFileSuite extends QueryTest with SharedSparkSession {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
class WholeTextFileV1Suite extends WholeTextFileSuite {
|
||||
override protected def sparkConf: SparkConf =
|
||||
super
|
||||
.sparkConf
|
||||
.set(SQLConf.USE_V1_SOURCE_LIST, "text")
|
||||
}
|
||||
|
||||
class WholeTextFileV2Suite extends WholeTextFileSuite {
|
||||
override protected def sparkConf: SparkConf =
|
||||
super
|
||||
.sparkConf
|
||||
.set(SQLConf.USE_V1_SOURCE_LIST, "")
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue