ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35840][SQL] Add apply() for a single field to YearMonthIntervalType and DayTimeIntervalType

Browse source 
### What changes were proposed in this pull request?
In the PR, I propose to add 2 new methods that accept one field and produce either `YearMonthIntervalType` or `DayTimeIntervalType`.

### Why are the changes needed?
To improve code maintenance.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
By existing test suites.

Closes #32997 from MaxGekk/ansi-interval-types-single-field.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Max Gekk 2021-06-21 14:15:33 +03:00
parent 1488ea9a8c
commit 37ef7bb98c
6 changed files with 22 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
View file

@ -173,18 +173,18 @@ object DataType {
private val otherTypes = {
Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
DayTimeIntervalType(DAY, DAY),
DayTimeIntervalType(DAY),
DayTimeIntervalType(DAY, HOUR),
DayTimeIntervalType(DAY, MINUTE),
DayTimeIntervalType(DAY, SECOND),
DayTimeIntervalType(HOUR, HOUR),
DayTimeIntervalType(HOUR),
DayTimeIntervalType(HOUR, MINUTE),
DayTimeIntervalType(HOUR, SECOND),
DayTimeIntervalType(MINUTE, MINUTE),
DayTimeIntervalType(MINUTE),
DayTimeIntervalType(MINUTE, SECOND),
DayTimeIntervalType(SECOND, SECOND),
YearMonthIntervalType(YEAR, YEAR),
YearMonthIntervalType(MONTH, MONTH),
DayTimeIntervalType(SECOND),
YearMonthIntervalType(YEAR),
YearMonthIntervalType(MONTH),
YearMonthIntervalType(YEAR, MONTH),
TimestampWithoutTZType)
.map(t => t.typeName -> t).toMap

1
sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
View file

@ -101,6 +101,7 @@ case object DayTimeIntervalType extends AbstractDataType {
val DEFAULT = DayTimeIntervalType(DAY, SECOND)
def apply(): DayTimeIntervalType = DEFAULT
def apply(field: Byte): DayTimeIntervalType = DayTimeIntervalType(field, field)
override private[sql] def defaultConcreteType: DataType = DEFAULT

1
sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
View file

@ -95,6 +95,7 @@ case object YearMonthIntervalType extends AbstractDataType {
val DEFAULT = YearMonthIntervalType(YEAR, MONTH)
def apply(): YearMonthIntervalType = DEFAULT
def apply(field: Byte): YearMonthIntervalType = YearMonthIntervalType(field, field)
override private[sql] def defaultConcreteType: DataType = DEFAULT

12
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
View file

@ -277,16 +277,16 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
test("SPARK-35726: Truncate java.time.Duration by fields of day-time interval type") {
val duration = Duration.ofSeconds(90061, 1000000023)
Seq((DayTimeIntervalType(DAY, DAY), 86400000000L, -86400000000L),
Seq((DayTimeIntervalType(DAY), 86400000000L, -86400000000L),
(DayTimeIntervalType(DAY, HOUR), 90000000000L, -90000000000L),
(DayTimeIntervalType(DAY, MINUTE), 90060000000L, -90060000000L),
(DayTimeIntervalType(DAY, SECOND), 90062000000L, -90062000001L),
(DayTimeIntervalType(HOUR, HOUR), 90000000000L, -90000000000L),
(DayTimeIntervalType(HOUR), 90000000000L, -90000000000L),
(DayTimeIntervalType(HOUR, MINUTE), 90060000000L, -90060000000L),
(DayTimeIntervalType(HOUR, SECOND), 90062000000L, -90062000001L),
(DayTimeIntervalType(MINUTE, MINUTE), 90060000000L, -90060000000L),
(DayTimeIntervalType(MINUTE), 90060000000L, -90060000000L),
(DayTimeIntervalType(MINUTE, SECOND), 90062000000L, -90062000001L),
(DayTimeIntervalType(SECOND, SECOND), 90062000000L, -90062000001L))
(DayTimeIntervalType(SECOND), 90062000000L, -90062000001L))
.foreach { case (dt, positive, negative) =>
assert(CatalystTypeConverters.createToCatalystConverter(dt)(duration) == positive)
assert(
@ -331,9 +331,9 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
}
test("SPARK-35769: Truncate java.time.Period by fields of year-month interval type") {
Seq(YearMonthIntervalType(YEAR, YEAR) -> 12,
Seq(YearMonthIntervalType(YEAR) -> 12,
YearMonthIntervalType(YEAR, MONTH) -> 13,
YearMonthIntervalType(MONTH, MONTH) -> 13)
YearMonthIntervalType(MONTH) -> 13)
.foreach { case (ym, value) =>
assert(CatalystTypeConverters.createToCatalystConverter(ym)(Period.of(1, 1, 0)) == value)
}

4
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
View file

@ -687,9 +687,9 @@ class CastSuite extends CastSuiteBase {
test("SPARK-35819: Support cast YearMonthIntervalType in different fields") {
val ym = cast(Literal.create("1-1"), YearMonthIntervalType(YEAR, MONTH))
Seq(YearMonthIntervalType(YEAR, YEAR) -> 12,
Seq(YearMonthIntervalType(YEAR) -> 12,
YearMonthIntervalType(YEAR, MONTH) -> 13,
YearMonthIntervalType(MONTH, MONTH) -> 13)
YearMonthIntervalType(MONTH) -> 13)
.foreach { case (dt, value) =>
checkEvaluation(cast(ym, dt), value)
}

12
sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeTestUtils.scala
View file

@ -53,21 +53,21 @@ object DataTypeTestUtils {
val numericTypeWithoutDecimal: Set[DataType] = integralType ++ Set(DoubleType, FloatType)
val dayTimeIntervalTypes: Seq[DayTimeIntervalType] = Seq(
DayTimeIntervalType(DAY, DAY),
DayTimeIntervalType(DAY),
DayTimeIntervalType(DAY, HOUR),
DayTimeIntervalType(DAY, MINUTE),
DayTimeIntervalType(DAY, SECOND),
DayTimeIntervalType(HOUR, HOUR),
DayTimeIntervalType(HOUR),
DayTimeIntervalType(HOUR, MINUTE),
DayTimeIntervalType(HOUR, SECOND),
DayTimeIntervalType(MINUTE, MINUTE),
DayTimeIntervalType(MINUTE),
DayTimeIntervalType(MINUTE, SECOND),
DayTimeIntervalType(SECOND, SECOND))
DayTimeIntervalType(SECOND))
val yearMonthIntervalTypes: Seq[YearMonthIntervalType] = Seq(
YearMonthIntervalType(YEAR, MONTH),
YearMonthIntervalType(YEAR, YEAR),
YearMonthIntervalType(MONTH, MONTH))
YearMonthIntervalType(YEAR),
YearMonthIntervalType(MONTH))
/**
* Instances of all [[NumericType]]s and [[CalendarIntervalType]]