ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35731][SQL][TESTS] Check all day-time interval types in arrow

Browse source 
### What changes were proposed in this pull request?
Check all day-time interval types in arrow.

### Why are the changes needed?
To improve test coverage.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Added UT.

Closes #33039 from AngersZhuuuu/SPARK-35731.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Angerszhuuuu 2021-06-23 23:38:41 +03:00 committed by Max Gekk
parent 2d3fa04e90
commit ad187227f1
1 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowWriterSuite.scala
View file

@ -31,6 +31,7 @@ class ArrowWriterSuite extends SparkFunSuite {
test("simple") { test("simple") {
def check(dt: DataType, data: Seq[Any], timeZoneId: String = null): Unit = { def check(dt: DataType, data: Seq[Any], timeZoneId: String = null): Unit = {
val avroDatatype = dt match { val avroDatatype = dt match {
case _: DayTimeIntervalType => DayTimeIntervalType()
case _: YearMonthIntervalType => YearMonthIntervalType() case _: YearMonthIntervalType => YearMonthIntervalType()
case tpe => tpe case tpe => tpe
} }
@ -83,8 +84,8 @@ class ArrowWriterSuite extends SparkFunSuite {
check(NullType, Seq(null, null, null)) check(NullType, Seq(null, null, null))
DataTypeTestUtils.yearMonthIntervalTypes DataTypeTestUtils.yearMonthIntervalTypes
.foreach(check(_, Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue))) .foreach(check(_, Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue)))
check(DayTimeIntervalType(), Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L), DataTypeTestUtils.dayTimeIntervalTypes.foreach(check(_,
(Long.MinValue + 808L))) Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L), (Long.MinValue + 808L))))
} }
test("long overflow for DayTimeIntervalType") test("long overflow for DayTimeIntervalType")
@ -114,6 +115,7 @@ class ArrowWriterSuite extends SparkFunSuite {
test("get multiple") { test("get multiple") {
def check(dt: DataType, data: Seq[Any], timeZoneId: String = null): Unit = { def check(dt: DataType, data: Seq[Any], timeZoneId: String = null): Unit = {
val avroDatatype = dt match { val avroDatatype = dt match {
case _: DayTimeIntervalType => DayTimeIntervalType()
case _: YearMonthIntervalType => YearMonthIntervalType() case _: YearMonthIntervalType => YearMonthIntervalType()
case tpe => tpe case tpe => tpe
} }
@ -154,8 +156,7 @@ class ArrowWriterSuite extends SparkFunSuite {
check(DateType, (0 until 10)) check(DateType, (0 until 10))
check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles") check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles")
DataTypeTestUtils.yearMonthIntervalTypes.foreach(check(_, (0 until 14))) DataTypeTestUtils.yearMonthIntervalTypes.foreach(check(_, (0 until 14)))
// TODO(SPARK-35731): Check all day-time interval types in arrow DataTypeTestUtils.dayTimeIntervalTypes.foreach(check(_, (-10 until 10).map(_ * 1000.toLong)))
check(DayTimeIntervalType(), (-10 until 10).map(_ * 1000.toLong))
} }
test("array") { test("array") {