[SPARK-35737][SQL] Parse day-time interval literals to tightest types
### What changes were proposed in this pull request? This PR add a feature which parse day-time interval literals to tightest type. ### Why are the changes needed? To comply with the ANSI behavior. For example, `INTERVAL '10 20:30' DAY TO MINUTE` should be parsed as `DayTimeIntervalType(DAY, MINUTE)` but not as `DayTimeIntervalType(DAY, SECOND)`. ### Does this PR introduce _any_ user-facing change? No because `DayTimeIntervalType` will be introduced in `3.2.0`. ### How was this patch tested? New tests. Closes #32892 from sarutak/tight-daytime-interval. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Kousuke Saruta
Max Gekk
parent
7978fdc97b
commit
439e94c171
|
|
@ -2357,9 +2357,14 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
|||
Literal(calendarInterval.months, YearMonthIntervalType)
|
||||
} else {
|
||||
assert(calendarInterval.months == 0)
|
||||
val strToFieldIndex = DayTimeIntervalType.dayTimeFields.map(i =>
|
||||
DayTimeIntervalType.fieldToString(i) -> i).toMap
|
||||
val fromUnit =
|
||||
ctx.errorCapturingUnitToUnitInterval.body.from.getText.toLowerCase(Locale.ROOT)
|
||||
val micros = IntervalUtils.getDuration(calendarInterval, TimeUnit.MICROSECONDS)
|
||||
// TODO(SPARK-35737): Parse day-time interval literals to tightest types
|
||||
Literal(micros, DayTimeIntervalType())
|
||||
val start = strToFieldIndex(fromUnit)
|
||||
val end = strToFieldIndex(toUnit)
|
||||
Literal(micros, DayTimeIntervalType(start, end))
|
||||
}
|
||||
} else {
|
||||
Literal(calendarInterval, CalendarIntervalType)
|
||||
|
|
|
|||
|
|
@ -363,7 +363,7 @@ struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
|
|||
-- !query
|
||||
select interval '20 15' day to hour
|
||||
-- !query schema
|
||||
struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '20 15' DAY TO HOUR:interval day to hour>
|
||||
-- !query output
|
||||
20 15:00:00.000000000
|
||||
|
||||
|
|
@ -371,7 +371,7 @@ struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '20 15:40' day to minute
|
||||
-- !query schema
|
||||
struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '20 15:40' DAY TO MINUTE:interval day to minute>
|
||||
-- !query output
|
||||
20 15:40:00.000000000
|
||||
|
||||
|
|
@ -387,7 +387,7 @@ struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '15:40' hour to minute
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '15:40' HOUR TO MINUTE:interval hour to minute>
|
||||
-- !query output
|
||||
0 15:40:00.000000000
|
||||
|
||||
|
|
@ -395,7 +395,7 @@ struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '15:40:32.99899999' hour to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '15:40:32.998999' HOUR TO SECOND:interval hour to second>
|
||||
-- !query output
|
||||
0 15:40:32.998999000
|
||||
|
||||
|
|
@ -403,7 +403,7 @@ struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '40:32.99899999' minute to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '40:32.998999' MINUTE TO SECOND:interval minute to second>
|
||||
-- !query output
|
||||
0 00:40:32.998999000
|
||||
|
||||
|
|
@ -411,7 +411,7 @@ struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '40:32' minute to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '40:32' MINUTE TO SECOND:interval minute to second>
|
||||
-- !query output
|
||||
0 00:40:32.000000000
|
||||
|
||||
|
|
|
|||
|
|
@ -357,7 +357,7 @@ struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
|
|||
-- !query
|
||||
select interval '20 15' day to hour
|
||||
-- !query schema
|
||||
struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '20 15' DAY TO HOUR:interval day to hour>
|
||||
-- !query output
|
||||
20 15:00:00.000000000
|
||||
|
||||
|
|
@ -365,7 +365,7 @@ struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '20 15:40' day to minute
|
||||
-- !query schema
|
||||
struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '20 15:40' DAY TO MINUTE:interval day to minute>
|
||||
-- !query output
|
||||
20 15:40:00.000000000
|
||||
|
||||
|
|
@ -381,7 +381,7 @@ struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '15:40' hour to minute
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '15:40' HOUR TO MINUTE:interval hour to minute>
|
||||
-- !query output
|
||||
0 15:40:00.000000000
|
||||
|
||||
|
|
@ -389,7 +389,7 @@ struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '15:40:32.99899999' hour to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '15:40:32.998999' HOUR TO SECOND:interval hour to second>
|
||||
-- !query output
|
||||
0 15:40:32.998999000
|
||||
|
||||
|
|
@ -397,7 +397,7 @@ struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '40:32.99899999' minute to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '40:32.998999' MINUTE TO SECOND:interval minute to second>
|
||||
-- !query output
|
||||
0 00:40:32.998999000
|
||||
|
||||
|
|
@ -405,7 +405,7 @@ struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
|
|||
-- !query
|
||||
select interval '40:32' minute to second
|
||||
-- !query schema
|
||||
struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
|
||||
struct<INTERVAL '40:32' MINUTE TO SECOND:interval minute to second>
|
||||
-- !query output
|
||||
0 00:40:32.000000000
|
||||
|
||||
|
|
|
|||
|
|
@ -4003,6 +4003,21 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
|
|||
}
|
||||
checkAnswer(sql(s"select /*+ REPARTITION(3, a) */ a b from values('123') t(a)"), Row("123"))
|
||||
}
|
||||
|
||||
test("SPARK-35737: Parse day-time interval literals to tightest types") {
|
||||
val dayToSecDF = spark.sql("SELECT INTERVAL '13 02:02:10' DAY TO SECOND")
|
||||
assert(dayToSecDF.schema.head.dataType === DayTimeIntervalType(0, 3))
|
||||
val dayToMinuteDF = spark.sql("SELECT INTERVAL '-2 13:00' DAY TO MINUTE")
|
||||
assert(dayToMinuteDF.schema.head.dataType === DayTimeIntervalType(0, 2))
|
||||
val dayToHourDF = spark.sql("SELECT INTERVAL '0 15' DAY TO HOUR")
|
||||
assert(dayToHourDF.schema.head.dataType === DayTimeIntervalType(0, 1))
|
||||
val hourToSecDF = spark.sql("SELECT INTERVAL '00:21:02.03' HOUR TO SECOND")
|
||||
assert(hourToSecDF.schema.head.dataType === DayTimeIntervalType(1, 3))
|
||||
val hourToMinuteDF = spark.sql("SELECT INTERVAL '01:02' HOUR TO MINUTE")
|
||||
assert(hourToMinuteDF.schema.head.dataType === DayTimeIntervalType(1, 2))
|
||||
val minuteToSecDF = spark.sql("SELECT INTERVAL '10:03.775808000' MINUTE TO SECOND")
|
||||
assert(minuteToSecDF.schema.head.dataType === DayTimeIntervalType(2, 3))
|
||||
}
|
||||
}
|
||||
|
||||
case class Foo(bar: Option[String])
|
||||
|
|
|
|||
Loading…
Reference in a new issue