[SPARK-36222][SQL] Step by days in the Sequence expression for dates

### What changes were proposed in this pull request?
The current implement of `Sequence` expression not support step by days for dates.
```
spark-sql> select sequence(date'2021-07-01', date'2021-07-10', interval '3' day);
Error in query: cannot resolve 'sequence(DATE '2021-07-01', DATE '2021-07-10', INTERVAL '3' DAY)' due to data type mismatch:
sequence uses the wrong parameter type. The parameter type must conform to:
1. The start and stop expressions must resolve to the same type.
2. If start and stop expressions resolve to the 'date' or 'timestamp' type
then the step expression must resolve to the 'interval' or
'interval year to month' or 'interval day to second' type,
otherwise to the same type as the start and stop expressions.
         ; line 1 pos 7;
'Project [unresolvedalias(sequence(2021-07-01, 2021-07-10, Some(INTERVAL '3' DAY), Some(Europe/Moscow)), None)]
+- OneRowRelation
```

### Why are the changes needed?
`DayTimeInterval` has day granularity should as step for dates.

### Does this PR introduce _any_ user-facing change?
'Yes'.
Sequence expression will supports step by `DayTimeInterval` has day granularity for dates.

### How was this patch tested?
New tests.

Closes #33439 from beliefer/SPARK-36222.

Authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
gengjiaan 2021-07-20 19:16:56 +03:00 committed by Max Gekk
parent bf680bf25a
commit c0d84e6cf1
2 changed files with 68 additions and 10 deletions

View file

@ -2574,7 +2574,8 @@ case class Sequence(
DayTimeIntervalType.acceptsType(stepType)
case DateType =>
stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepType) ||
YearMonthIntervalType.acceptsType(stepType)
YearMonthIntervalType.acceptsType(stepType) ||
DayTimeIntervalType.acceptsType(stepType)
case _: IntegralType =>
stepOpt.isEmpty || stepType.sameType(startType)
case _ => false
@ -2626,8 +2627,10 @@ case class Sequence(
case DateType =>
if (stepOpt.isEmpty || CalendarIntervalType.acceptsType(stepOpt.get.dataType)) {
new TemporalSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
} else {
} else if (YearMonthIntervalType.acceptsType(stepOpt.get.dataType)) {
new PeriodSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
} else {
new DurationSequenceImpl[Int](IntegerType, start.dataType, MICROS_PER_DAY, _.toInt, zoneId)
}
}
@ -2807,15 +2810,19 @@ object Sequence {
val intervalType: DataType = DayTimeIntervalType()
def splitStep(input: Any): (Int, Int, Long) = {
(0, 0, input.asInstanceOf[Long])
val duration = input.asInstanceOf[Long]
val days = IntervalUtils.getDays(duration)
val micros = duration - days * MICROS_PER_DAY
(0, days, micros)
}
def stepSplitCode(
stepMonths: String, stepDays: String, stepMicros: String, step: String): String = {
s"""
|final int $stepMonths = 0;
|final int $stepDays = 0;
|final long $stepMicros = $step;
|final int $stepDays =
| (int) org.apache.spark.sql.catalyst.util.IntervalUtils.getDays($step);
|final long $stepMicros = $step - $stepDays * ${MICROS_PER_DAY}L;
""".stripMargin
}
}

View file

@ -919,6 +919,16 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Date.valueOf("2020-11-01"),
Date.valueOf("2022-04-01")))
checkEvaluation(new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(fromDayTimeString("3 0:0:0"))),
Seq(
Date.valueOf("2021-07-01"),
Date.valueOf("2021-07-04"),
Date.valueOf("2021-07-07"),
Date.valueOf("2021-07-10")))
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("1970-01-02")),
@ -1092,6 +1102,52 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Date.valueOf("2020-11-01"),
Date.valueOf("2022-04-01")))
checkEvaluation(new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofDays(3))),
Seq(
Date.valueOf("2021-07-01"),
Date.valueOf("2021-07-04"),
Date.valueOf("2021-07-07"),
Date.valueOf("2021-07-10")))
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofHours(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofMinutes(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofSeconds(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2021-07-01")),
Literal(Date.valueOf("2021-07-10")),
Literal(Duration.ofMillis(3))),
EmptyRow,
"sequence step must be an interval day to second of day granularity" +
" if start and end values are dates")
checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Literal(Date.valueOf("2018-01-01")),
@ -1108,11 +1164,6 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Literal(Period.ofMonths(-1))),
EmptyRow,
s"sequence boundaries: 0 to 2678400000000 by -1")
assert(Sequence(
Cast(Literal("2011-03-01"), DateType),
Cast(Literal("2011-04-01"), DateType),
Option(Literal(Duration.ofHours(1)))).checkInputDataTypes().isFailure)
}
}