[SPARK-36428][SQL] the seconds parameter of make_timestamp should accept integer type
### What changes were proposed in this pull request?
With ANSI mode, `SELECT make_timestamp(1, 1, 1, 1, 1, 1)` fails, because the 'seconds' parameter needs to be of type DECIMAL(8,6), and INT can't be implicitly casted to DECIMAL(8,6) under ANSI mode.
```
org.apache.spark.sql.AnalysisException
cannot resolve 'make_timestamp(1, 1, 1, 1, 1, 1)' due to data type mismatch: argument 6 requires decimal(8,6) type, however, '1' is of int type.; line 1 pos 7
```
We should update the function `make_timestamp` to allow integer type 'seconds' parameter.
### Why are the changes needed?
Make `make_timestamp` could accepts integer as 'seconds' parameter.
### Does this PR introduce _any_ user-facing change?
'Yes'.
`make_timestamp` could accepts integer as 'seconds' parameter.
### How was this patch tested?
New tests.
Closes #33665 from beliefer/SPARK-36428.
Lead-authored-by: gengjiaan <gengjiaan@360.cn>
Co-authored-by: Jiaan Geng <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7d82336734
)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
ca91292cf0
commit
eb840578f7
|
@ -2494,8 +2494,9 @@ case class MakeTimestampLTZ(
|
||||||
* day - the day-of-month to represent, from 1 to 31
|
* day - the day-of-month to represent, from 1 to 31
|
||||||
* hour - the hour-of-day to represent, from 0 to 23
|
* hour - the hour-of-day to represent, from 0 to 23
|
||||||
* min - the minute-of-hour to represent, from 0 to 59
|
* min - the minute-of-hour to represent, from 0 to 59
|
||||||
* sec - the second-of-minute and its micro-fraction to represent, from
|
* sec - the second-of-minute and its micro-fraction to represent, from 0 to 60.
|
||||||
0 to 60. If the sec argument equals to 60, the seconds field is set
|
The value can be either an integer like 13 , or a fraction like 13.123.
|
||||||
|
If the sec argument equals to 60, the seconds field is set
|
||||||
to 0 and 1 minute is added to the final timestamp.
|
to 0 and 1 minute is added to the final timestamp.
|
||||||
* timezone - the time zone identifier. For example, CET, UTC and etc.
|
* timezone - the time zone identifier. For example, CET, UTC and etc.
|
||||||
""",
|
""",
|
||||||
|
@ -2507,6 +2508,8 @@ case class MakeTimestampLTZ(
|
||||||
2014-12-27 21:30:45.887
|
2014-12-27 21:30:45.887
|
||||||
> SELECT _FUNC_(2019, 6, 30, 23, 59, 60);
|
> SELECT _FUNC_(2019, 6, 30, 23, 59, 60);
|
||||||
2019-07-01 00:00:00
|
2019-07-01 00:00:00
|
||||||
|
> SELECT _FUNC_(2019, 6, 30, 23, 59, 1);
|
||||||
|
2019-06-30 23:59:01
|
||||||
> SELECT _FUNC_(2019, 13, 1, 10, 11, 12, 'PST');
|
> SELECT _FUNC_(2019, 13, 1, 10, 11, 12, 'PST');
|
||||||
NULL
|
NULL
|
||||||
> SELECT _FUNC_(null, 7, 22, 15, 30, 0);
|
> SELECT _FUNC_(null, 7, 22, 15, 30, 0);
|
||||||
|
@ -2556,13 +2559,20 @@ case class MakeTimestamp(
|
||||||
// Accept `sec` as DecimalType to avoid loosing precision of microseconds while converting
|
// Accept `sec` as DecimalType to avoid loosing precision of microseconds while converting
|
||||||
// them to the fractional part of `sec`.
|
// them to the fractional part of `sec`.
|
||||||
override def inputTypes: Seq[AbstractDataType] =
|
override def inputTypes: Seq[AbstractDataType] =
|
||||||
Seq(IntegerType, IntegerType, IntegerType, IntegerType, IntegerType, DecimalType(8, 6)) ++
|
Seq(IntegerType, IntegerType, IntegerType, IntegerType, IntegerType,
|
||||||
timezone.map(_ => StringType)
|
TypeCollection(DecimalType(8, 6), IntegerType, NullType)) ++ timezone.map(_ => StringType)
|
||||||
override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
|
override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
|
||||||
|
|
||||||
override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
|
override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
|
||||||
copy(timeZoneId = Option(timeZoneId))
|
copy(timeZoneId = Option(timeZoneId))
|
||||||
|
|
||||||
|
private lazy val toDecimal = sec.dataType match {
|
||||||
|
case DecimalType() =>
|
||||||
|
(secEval: Any) => secEval.asInstanceOf[Decimal]
|
||||||
|
case IntegerType =>
|
||||||
|
(secEval: Any) => Decimal(BigDecimal(secEval.asInstanceOf[Int]), 8, 6)
|
||||||
|
}
|
||||||
|
|
||||||
private def toMicros(
|
private def toMicros(
|
||||||
year: Int,
|
year: Int,
|
||||||
month: Int,
|
month: Int,
|
||||||
|
@ -2617,7 +2627,7 @@ case class MakeTimestamp(
|
||||||
day.asInstanceOf[Int],
|
day.asInstanceOf[Int],
|
||||||
hour.asInstanceOf[Int],
|
hour.asInstanceOf[Int],
|
||||||
min.asInstanceOf[Int],
|
min.asInstanceOf[Int],
|
||||||
sec.asInstanceOf[Decimal],
|
toDecimal(sec),
|
||||||
zid)
|
zid)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2625,6 +2635,7 @@ case class MakeTimestamp(
|
||||||
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
|
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
|
||||||
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
|
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
|
||||||
val d = Decimal.getClass.getName.stripSuffix("$")
|
val d = Decimal.getClass.getName.stripSuffix("$")
|
||||||
|
val decimalValue = ctx.freshName("decimalValue")
|
||||||
val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
|
val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
|
||||||
nullSafeCodeGen(ctx, ev, (year, month, day, hour, min, secAndNanos, timezone) => {
|
nullSafeCodeGen(ctx, ev, (year, month, day, hour, min, secAndNanos, timezone) => {
|
||||||
val zoneId = timezone.map(tz => s"$dtu.getZoneId(${tz}.toString())").getOrElse(zid)
|
val zoneId = timezone.map(tz => s"$dtu.getZoneId(${tz}.toString())").getOrElse(zid)
|
||||||
|
@ -2636,11 +2647,21 @@ case class MakeTimestamp(
|
||||||
} else {
|
} else {
|
||||||
s"${ev.value} = $dtu.localDateTimeToMicros(ldt);"
|
s"${ev.value} = $dtu.localDateTimeToMicros(ldt);"
|
||||||
}
|
}
|
||||||
|
val toDecimalCode = sec.dataType match {
|
||||||
|
case DecimalType() =>
|
||||||
|
s"org.apache.spark.sql.types.Decimal $decimalValue = $secAndNanos;"
|
||||||
|
case IntegerType =>
|
||||||
|
s"""
|
||||||
|
|org.apache.spark.sql.types.Decimal $decimalValue =
|
||||||
|
|$d$$.MODULE$$.apply(new java.math.BigDecimal($secAndNanos), 8, 6);
|
||||||
|
""".stripMargin
|
||||||
|
}
|
||||||
s"""
|
s"""
|
||||||
try {
|
try {
|
||||||
org.apache.spark.sql.types.Decimal secFloor = $secAndNanos.floor();
|
$toDecimalCode
|
||||||
|
org.apache.spark.sql.types.Decimal secFloor = $decimalValue.floor();
|
||||||
org.apache.spark.sql.types.Decimal nanosPerSec = $d$$.MODULE$$.apply(1000000000L, 10, 0);
|
org.apache.spark.sql.types.Decimal nanosPerSec = $d$$.MODULE$$.apply(1000000000L, 10, 0);
|
||||||
int nanos = (($secAndNanos.$$minus(secFloor)).$$times(nanosPerSec)).toInt();
|
int nanos = (($decimalValue.$$minus(secFloor)).$$times(nanosPerSec)).toInt();
|
||||||
int seconds = secFloor.toInt();
|
int seconds = secFloor.toInt();
|
||||||
java.time.LocalDateTime ldt;
|
java.time.LocalDateTime ldt;
|
||||||
if (seconds == 60) {
|
if (seconds == 60) {
|
||||||
|
|
|
@ -1222,6 +1222,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-08-12 00:00:58.000001"))
|
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-08-12 00:00:58.000001"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Seq(true, false).foreach { ansi =>
|
||||||
|
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
|
||||||
|
val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12),
|
||||||
|
Literal(0), Literal(0), Literal(1))
|
||||||
|
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-08-12 00:00:01"))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,10 @@ select localtimestamp() = localtimestamp();
|
||||||
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678);
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678);
|
||||||
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678, 'CET');
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678, 'CET');
|
||||||
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007);
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007);
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1);
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60);
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61);
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null);
|
||||||
|
|
||||||
-- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS that always create timestamp_ltz
|
-- [SPARK-31710] TIMESTAMP_SECONDS, TIMESTAMP_MILLISECONDS and TIMESTAMP_MICROSECONDS that always create timestamp_ltz
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null);
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null);
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 82
|
-- Number of queries: 86
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -107,6 +107,39 @@ java.time.DateTimeException
|
||||||
The fraction of sec must be zero. Valid range is [0, 60].
|
The fraction of sec must be zero. Valid range is [0, 60].
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 1):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:01:01
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 60):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:02:00
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61)
|
||||||
|
-- !query schema
|
||||||
|
struct<>
|
||||||
|
-- !query output
|
||||||
|
java.time.DateTimeException
|
||||||
|
Invalid value for SecondOfMinute (valid values 0 - 59): 61
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, NULL):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
||||||
-- !query schema
|
-- !query schema
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 159
|
-- Number of queries: 163
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -758,6 +758,38 @@ struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp>
|
||||||
NULL
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 1):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:01:01
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 60):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:02:00
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 61):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, NULL):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
||||||
-- !query schema
|
-- !query schema
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 82
|
-- Number of queries: 86
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -100,6 +100,38 @@ struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp>
|
||||||
NULL
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 1):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:01:01
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 60):timestamp>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:02:00
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 61):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, NULL):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
||||||
-- !query schema
|
-- !query schema
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 82
|
-- Number of queries: 86
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -107,6 +107,39 @@ java.time.DateTimeException
|
||||||
The fraction of sec must be zero. Valid range is [0, 60].
|
The fraction of sec must be zero. Valid range is [0, 60].
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 1):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:01:01
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 60):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:02:00
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61)
|
||||||
|
-- !query schema
|
||||||
|
struct<>
|
||||||
|
-- !query output
|
||||||
|
java.time.DateTimeException
|
||||||
|
Invalid value for SecondOfMinute (valid values 0 - 59): 61
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, NULL):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
||||||
-- !query schema
|
-- !query schema
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 82
|
-- Number of queries: 86
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -100,6 +100,38 @@ struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp_ntz>
|
||||||
NULL
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 1)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 1):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:01:01
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 60)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 60):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
0001-01-01 01:02:00
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, 61)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, 61):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(1, 1, 1, 1, 1, null)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(1, 1, 1, 1, 1, NULL):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
select TIMESTAMP_SECONDS(1230219000),TIMESTAMP_SECONDS(-1230219000),TIMESTAMP_SECONDS(null)
|
||||||
-- !query schema
|
-- !query schema
|
||||||
|
|
Loading…
Reference in a new issue