ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35728][SPARK-35778][SQL][TESTS] Check multiply/divide of day-time and year-month interval of any fields by a numeric

Browse source 
### What changes were proposed in this pull request?
[SPARK-35728](https://issues.apache.org/jira/browse/SPARK-35728): Add test case to check multiply/divide of day-time
intervals of any fields by numeric
[SPARK-35778](https://issues.apache.org/jira/browse/SPARK-35778): Add test case to check multiply/divide of year-month intervals of any fields by numeric

### Why are the changes needed?
Improve test coverage

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Add ut tests

Lead-authored-by: Lei Peng <peng.8leigmail.com>
Co-authored-by: AngersZhuuuu <angers.zhugmail.com>

Closes #33080 from Peng-Lei/SPARK-35728-35778.

Lead-authored-by: PengLei <peng.8lei@gmail.com>
Co-authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Co-authored-by: PengLei <18066542445@189.cn>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
PengLei 2021-06-28 13:35:54 +03:00 committed by Max Gekk
parent 108635af17
commit 356aef48b8
1 changed files with 131 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

136
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
View file

@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils}
import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval} import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType} import org.apache.spark.sql.types.{DataTypeTestUtils, DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType}
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes, yearMonthIntervalTypes} import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes, yearMonthIntervalTypes}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@ -325,7 +325,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkException(days = Int.MaxValue) checkException(days = Int.MaxValue)
} }
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
test("SPARK-34824: multiply year-month interval by numeric") { test("SPARK-34824: multiply year-month interval by numeric") {
Seq( Seq(
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -361,7 +360,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
} }
// TODO(SPARK-35728): Check multiply/divide of day-time intervals of any fields by numeric
test("SPARK-34850: multiply day-time interval by numeric") { test("SPARK-34850: multiply day-time interval by numeric") {
Seq( Seq(
(Duration.ofHours(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Duration.ofHours(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -397,7 +395,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
} }
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
test("SPARK-34868: divide year-month interval by numeric") { test("SPARK-34868: divide year-month interval by numeric") {
Seq( Seq(
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -433,7 +430,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
} }
// TODO(SPARK-35728): Check multiply/divide of day-time intervals of any fields by numeric
test("SPARK-34875: divide day-time interval by numeric") { test("SPARK-34875: divide day-time interval by numeric") {
Seq( Seq(
(Duration.ofDays(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Duration.ofDays(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -535,4 +531,134 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkImplicitEvaluation(MakeYMInterval(Literal(1d), Literal(-1L)), 11) checkImplicitEvaluation(MakeYMInterval(Literal(1d), Literal(-1L)), 11)
checkImplicitEvaluation(MakeYMInterval(Literal(1.1), Literal(-1L)), 11) checkImplicitEvaluation(MakeYMInterval(Literal(1.1), Literal(-1L)), 11)
} }
test("SPARK-35728: Check multiply/divide of day-time intervals of any fields by numeric") {
Seq(
((Duration.ofMinutes(0), 10),
Array(Duration.ofDays(0), Duration.ofHours(0), Duration.ofMinutes(0),
Duration.ofSeconds(0), Duration.ofHours(0), Duration.ofMinutes(0),
Duration.ofSeconds(0), Duration.ofMinutes(0), Duration.ofSeconds(0),
Duration.ofSeconds(0)),
Array(Duration.ofDays(0), Duration.ofHours(0), Duration.ofMinutes(0),
Duration.ofSeconds(0), Duration.ofHours(0), Duration.ofMinutes(0),
Duration.ofSeconds(0), Duration.ofMinutes(0), Duration.ofSeconds(0),
Duration.ofSeconds(0))),
((Duration.ofSeconds(86400 + 3600 + 60 + 1), 1L),
Array(Duration.ofSeconds(86400), Duration.ofSeconds(90000),
Duration.ofSeconds(90060), Duration.ofSeconds(90061),
Duration.ofSeconds(90000), Duration.ofSeconds(90060),
Duration.ofSeconds(90061), Duration.ofSeconds(90060),
Duration.ofSeconds(90061), Duration.ofSeconds(90061)),
Array(Duration.ofSeconds(86400), Duration.ofSeconds(90000),
Duration.ofSeconds(90060), Duration.ofSeconds(90061),
Duration.ofSeconds(90000), Duration.ofSeconds(90060),
Duration.ofSeconds(90061), Duration.ofSeconds(90060),
Duration.ofSeconds(90061), Duration.ofSeconds(90061))),
((Duration.ofSeconds(86400 + 3600 + 60 + 1), -1.toByte),
Array(Duration.ofSeconds(-86400), Duration.ofSeconds(-90000),
Duration.ofSeconds(-90060), Duration.ofSeconds(-90061),
Duration.ofSeconds(-90000), Duration.ofSeconds(-90060),
Duration.ofSeconds(-90061), Duration.ofSeconds(-90060),
Duration.ofSeconds(-90061), Duration.ofSeconds(-90061)),
Array(Duration.ofSeconds(-86400), Duration.ofSeconds(-90000),
Duration.ofSeconds(-90060), Duration.ofSeconds(-90061),
Duration.ofSeconds(-90000), Duration.ofSeconds(-90060),
Duration.ofSeconds(-90061), Duration.ofSeconds(-90060),
Duration.ofSeconds(-90061), Duration.ofSeconds(-90061))),
((Duration.ofSeconds(86400 + 3600 + 60 + 1), 0.3d),
Array(Duration.ofSeconds(288000), Duration.ofSeconds(300000),
Duration.ofSeconds(300200), Duration.ofNanos(300203333333000L),
Duration.ofSeconds(300000), Duration.ofSeconds(300200),
Duration.ofNanos(300203333333000L), Duration.ofSeconds(300200),
Duration.ofNanos(300203333333000L), Duration.ofNanos(300203333333000L)),
Array(Duration.ofSeconds(25920), Duration.ofSeconds(27000),
Duration.ofSeconds(27018), Duration.ofMillis(27018300),
Duration.ofSeconds(27000), Duration.ofSeconds(27018),
Duration.ofMillis(27018300), Duration.ofSeconds(27018),
Duration.ofMillis(27018300), Duration.ofMillis(27018300))),
((Duration.of(-1000, ChronoUnit.MICROS), 0.3f),
Array(Duration.ofSeconds(0), Duration.ofSeconds(0),
Duration.ofSeconds(0), Duration.ofNanos(-3333000L),
Duration.ofSeconds(0), Duration.ofSeconds(0),
Duration.ofNanos(-3333000L), Duration.ofSeconds(0),
Duration.ofNanos(-3333000L), Duration.ofNanos(-3333000L)),
Array(Duration.ofSeconds(0), Duration.ofSeconds(0),
Duration.ofSeconds(0), Duration.ofNanos(-300000),
Duration.ofSeconds(0), Duration.ofSeconds(0),
Duration.ofNanos(-300000), Duration.ofSeconds(0),
Duration.ofNanos(-300000), Duration.ofNanos(-300000))),
((Duration.ofDays(9999), 0.0001d),
Array(Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L)),
Array(Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360))),
((Duration.ofDays(9999), BigDecimal(0.0001)),
Array(Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L),
Duration.ofSeconds(8639136000000L), Duration.ofSeconds(8639136000000L)),
Array(Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360),
Duration.ofMillis(86391360), Duration.ofMillis(86391360)))
).foreach { case ((duration, num), divideExpected, multiplyExpected) =>
DataTypeTestUtils.dayTimeIntervalTypes.zip(divideExpected)
.foreach { case (dt, result) =>
checkEvaluation(DivideDTInterval(Literal.create(duration, dt), Literal(num)), result)
}
DataTypeTestUtils.dayTimeIntervalTypes.zip(multiplyExpected)
.foreach { case (dt, result) =>
checkEvaluation(MultiplyDTInterval(Literal.create(duration, dt), Literal(num)), result)
}
}
}
test("SPARK-35778: Check multiply/divide of year-month intervals of any fields by numeric") {
Seq(
((Period.ofMonths(0), 10),
Array(Period.ofMonths(0), Period.ofMonths(0), Period.ofMonths(0)),
Array(Period.ofMonths(0), Period.ofMonths(0), Period.ofMonths(0))),
((Period.ofMonths(13), 1),
Array(Period.ofMonths(13), Period.ofMonths(12), Period.ofMonths(13)),
Array(Period.ofMonths(13), Period.ofMonths(12), Period.ofMonths(13))),
((Period.ofMonths(-200), 1),
Array(Period.ofMonths(-200), Period.ofMonths(-192), Period.ofMonths(-200)),
Array(Period.ofMonths(-200), Period.ofMonths(-192), Period.ofMonths(-200))),
((Period.ofYears(100), -1.toByte),
Array(Period.ofMonths(-1200), Period.ofMonths(-1200), Period.ofMonths(-1200)),
Array(Period.ofMonths(-1200), Period.ofMonths(-1200), Period.ofMonths(-1200))),
((Period.ofYears(1), 2.toShort),
Array(Period.ofMonths(6), Period.ofMonths(6), Period.ofMonths(6)),
Array(Period.ofMonths(24), Period.ofMonths(24), Period.ofMonths(24))),
((Period.ofYears(-1), -3),
Array(Period.ofMonths(4), Period.ofMonths(4), Period.ofMonths(4)),
Array(Period.ofMonths(36), Period.ofMonths(36), Period.ofMonths(36))),
((Period.ofMonths(-1000), 0.5f),
Array(Period.ofMonths(-2000), Period.ofMonths(-1992), Period.ofMonths(-2000)),
Array(Period.ofMonths(-500), Period.ofMonths(-498), Period.ofMonths(-500))),
((Period.ofYears(1000), 100d),
Array(Period.ofYears(10), Period.ofYears(10), Period.ofYears(10)),
Array(Period.ofMonths(1200000), Period.ofMonths(1200000), Period.ofMonths(1200000))),
((Period.ofMonths(2), BigDecimal(0.1)),
Array(Period.ofMonths(20), Period.ofMonths(0), Period.ofMonths(20)),
Array(Period.ofMonths(0), Period.ofMonths(0), Period.ofMonths(0)))
).foreach { case ((period, num), divideExpected, multiplyExpected) =>
DataTypeTestUtils.yearMonthIntervalTypes.zip(divideExpected)
.foreach { case (dt, result) =>
checkEvaluation(DivideYMInterval(Literal.create(period, dt), Literal(num)), result)
}
DataTypeTestUtils.yearMonthIntervalTypes.zip(multiplyExpected)
.foreach { case (dt, result) =>
checkEvaluation(MultiplyYMInterval(Literal.create(period, dt), Literal(num)), result)
}
}
}
} }