[SPARK-36083][SQL] make_timestamp: return different result based on the default timestamp type
### What changes were proposed in this pull request?
The SQL function MAKE_TIMESTAMP should return different results based on the default timestamp type:
* when "spark.sql.timestampType" is TIMESTAMP_NTZ, return TimestampNTZType literal
* when "spark.sql.timestampType" is TIMESTAMP_LTZ, return TimestampType literal
### Why are the changes needed?
As "spark.sql.timestampType" sets the default timestamp type, the make_timestamp function should behave consistently with it.
### Does this PR introduce _any_ user-facing change?
Yes, when the value of "spark.sql.timestampType" is TIMESTAMP_NTZ, the result type of `MAKE_TIMESTAMP` is of TIMESTAMP_NTZ type.
### How was this patch tested?
Unit test
Closes #33290 from gengliangwang/mkTS.
Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
(cherry picked from commit 17ddcc9e82
)
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
9b8683ec88
commit
09e5bbdfbe
|
@ -2286,7 +2286,8 @@ case class MakeDate(
|
||||||
|
|
||||||
// scalastyle:off line.size.limit
|
// scalastyle:off line.size.limit
|
||||||
@ExpressionDescription(
|
@ExpressionDescription(
|
||||||
usage = "_FUNC_(year, month, day, hour, min, sec[, timezone]) - Create timestamp from year, month, day, hour, min, sec and timezone fields.",
|
usage = "_FUNC_(year, month, day, hour, min, sec[, timezone]) - Create timestamp from year, month, day, hour, min, sec and timezone fields. " +
|
||||||
|
"The result data type is consistent with the value of configuration `spark.sql.timestampType`",
|
||||||
arguments = """
|
arguments = """
|
||||||
Arguments:
|
Arguments:
|
||||||
* year - the year to represent, from 1 to 9999
|
* year - the year to represent, from 1 to 9999
|
||||||
|
@ -2324,7 +2325,8 @@ case class MakeTimestamp(
|
||||||
sec: Expression,
|
sec: Expression,
|
||||||
timezone: Option[Expression] = None,
|
timezone: Option[Expression] = None,
|
||||||
timeZoneId: Option[String] = None,
|
timeZoneId: Option[String] = None,
|
||||||
failOnError: Boolean = SQLConf.get.ansiEnabled)
|
failOnError: Boolean = SQLConf.get.ansiEnabled,
|
||||||
|
override val dataType: DataType = SQLConf.get.timestampType)
|
||||||
extends SeptenaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
|
extends SeptenaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
|
||||||
with NullIntolerant {
|
with NullIntolerant {
|
||||||
|
|
||||||
|
@ -2335,7 +2337,8 @@ case class MakeTimestamp(
|
||||||
hour: Expression,
|
hour: Expression,
|
||||||
min: Expression,
|
min: Expression,
|
||||||
sec: Expression) = {
|
sec: Expression) = {
|
||||||
this(year, month, day, hour, min, sec, None, None, SQLConf.get.ansiEnabled)
|
this(year, month, day, hour, min, sec, None, None, SQLConf.get.ansiEnabled,
|
||||||
|
SQLConf.get.timestampType)
|
||||||
}
|
}
|
||||||
|
|
||||||
def this(
|
def this(
|
||||||
|
@ -2346,7 +2349,8 @@ case class MakeTimestamp(
|
||||||
min: Expression,
|
min: Expression,
|
||||||
sec: Expression,
|
sec: Expression,
|
||||||
timezone: Expression) = {
|
timezone: Expression) = {
|
||||||
this(year, month, day, hour, min, sec, Some(timezone), None, SQLConf.get.ansiEnabled)
|
this(year, month, day, hour, min, sec, Some(timezone), None, SQLConf.get.ansiEnabled,
|
||||||
|
SQLConf.get.timestampType)
|
||||||
}
|
}
|
||||||
|
|
||||||
override def children: Seq[Expression] = Seq(year, month, day, hour, min, sec) ++ timezone
|
override def children: Seq[Expression] = Seq(year, month, day, hour, min, sec) ++ timezone
|
||||||
|
@ -2355,7 +2359,6 @@ case class MakeTimestamp(
|
||||||
override def inputTypes: Seq[AbstractDataType] =
|
override def inputTypes: Seq[AbstractDataType] =
|
||||||
Seq(IntegerType, IntegerType, IntegerType, IntegerType, IntegerType, DecimalType(8, 6)) ++
|
Seq(IntegerType, IntegerType, IntegerType, IntegerType, IntegerType, DecimalType(8, 6)) ++
|
||||||
timezone.map(_ => StringType)
|
timezone.map(_ => StringType)
|
||||||
override def dataType: DataType = TimestampType
|
|
||||||
override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
|
override def nullable: Boolean = if (failOnError) children.exists(_.nullable) else true
|
||||||
|
|
||||||
override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
|
override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
|
||||||
|
@ -2388,7 +2391,11 @@ case class MakeTimestamp(
|
||||||
} else {
|
} else {
|
||||||
LocalDateTime.of(year, month, day, hour, min, seconds, nanos)
|
LocalDateTime.of(year, month, day, hour, min, seconds, nanos)
|
||||||
}
|
}
|
||||||
|
if (dataType == TimestampType) {
|
||||||
instantToMicros(ldt.atZone(zoneId).toInstant)
|
instantToMicros(ldt.atZone(zoneId).toInstant)
|
||||||
|
} else {
|
||||||
|
localDateTimeToMicros(ldt)
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
case _: DateTimeException if !failOnError => null
|
case _: DateTimeException if !failOnError => null
|
||||||
}
|
}
|
||||||
|
@ -2422,6 +2429,14 @@ case class MakeTimestamp(
|
||||||
val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
|
val failOnErrorBranch = if (failOnError) "throw e;" else s"${ev.isNull} = true;"
|
||||||
nullSafeCodeGen(ctx, ev, (year, month, day, hour, min, secAndNanos, timezone) => {
|
nullSafeCodeGen(ctx, ev, (year, month, day, hour, min, secAndNanos, timezone) => {
|
||||||
val zoneId = timezone.map(tz => s"$dtu.getZoneId(${tz}.toString())").getOrElse(zid)
|
val zoneId = timezone.map(tz => s"$dtu.getZoneId(${tz}.toString())").getOrElse(zid)
|
||||||
|
val toMicrosCode = if (dataType == TimestampType) {
|
||||||
|
s"""
|
||||||
|
|java.time.Instant instant = ldt.atZone($zoneId).toInstant();
|
||||||
|
|${ev.value} = $dtu.instantToMicros(instant);
|
||||||
|
|""".stripMargin
|
||||||
|
} else {
|
||||||
|
s"${ev.value} = $dtu.localDateTimeToMicros(ldt);"
|
||||||
|
}
|
||||||
s"""
|
s"""
|
||||||
try {
|
try {
|
||||||
org.apache.spark.sql.types.Decimal secFloor = $secAndNanos.floor();
|
org.apache.spark.sql.types.Decimal secFloor = $secAndNanos.floor();
|
||||||
|
@ -2439,8 +2454,7 @@ case class MakeTimestamp(
|
||||||
} else {
|
} else {
|
||||||
ldt = java.time.LocalDateTime.of($year, $month, $day, $hour, $min, seconds, nanos);
|
ldt = java.time.LocalDateTime.of($year, $month, $day, $hour, $min, seconds, nanos);
|
||||||
}
|
}
|
||||||
java.time.Instant instant = ldt.atZone($zoneId).toInstant();
|
$toMicrosCode
|
||||||
${ev.value} = $dtu.instantToMicros(instant);
|
|
||||||
} catch (java.time.DateTimeException e) {
|
} catch (java.time.DateTimeException e) {
|
||||||
$failOnErrorBranch
|
$failOnErrorBranch
|
||||||
}"""
|
}"""
|
||||||
|
|
|
@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
|
||||||
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
|
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
|
||||||
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
|
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
|
import org.apache.spark.sql.internal.SQLConf.TimestampTypes
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
||||||
|
@ -1131,8 +1132,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test("creating values of TimestampType via make_timestamp") {
|
test("creating values of Timestamp/TimestampNTZ via make_timestamp") {
|
||||||
val expected = Timestamp.valueOf("2013-7-15 8:15:23.5")
|
Seq(TimestampTypes.TIMESTAMP_NTZ, TimestampTypes.TIMESTAMP_LTZ).foreach { tsType =>
|
||||||
|
def expectedAnswer(s: String): Any = tsType match {
|
||||||
|
case TimestampTypes.TIMESTAMP_NTZ => LocalDateTime.parse(s.replace(" ", "T"))
|
||||||
|
case TimestampTypes.TIMESTAMP_LTZ => Timestamp.valueOf(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> tsType.toString) {
|
||||||
|
val expected = expectedAnswer("2013-07-15 08:15:23.5")
|
||||||
|
|
||||||
Seq(true, false).foreach { ansi =>
|
Seq(true, false).foreach { ansi =>
|
||||||
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
|
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
|
||||||
|
@ -1145,7 +1153,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(makeTimestampExpr.copy(day = Literal.create(null, IntegerType)), null)
|
checkEvaluation(makeTimestampExpr.copy(day = Literal.create(null, IntegerType)), null)
|
||||||
checkEvaluation(makeTimestampExpr.copy(hour = Literal.create(null, IntegerType)), null)
|
checkEvaluation(makeTimestampExpr.copy(hour = Literal.create(null, IntegerType)), null)
|
||||||
checkEvaluation(makeTimestampExpr.copy(min = Literal.create(null, IntegerType)), null)
|
checkEvaluation(makeTimestampExpr.copy(min = Literal.create(null, IntegerType)), null)
|
||||||
checkEvaluation(makeTimestampExpr.copy(sec = Literal.create(null, DecimalType(8, 6))), null)
|
checkEvaluation(makeTimestampExpr.copy(sec = Literal.create(null, DecimalType(8, 6))),
|
||||||
|
null)
|
||||||
checkEvaluation(makeTimestampExpr.copy(timezone = None), expected)
|
checkEvaluation(makeTimestampExpr.copy(timezone = None), expected)
|
||||||
|
|
||||||
Seq(
|
Seq(
|
||||||
|
@ -1170,12 +1179,12 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkExceptionInExpression[DateTimeException](makeTimestampExpr.copy(sec = Literal(
|
checkExceptionInExpression[DateTimeException](makeTimestampExpr.copy(sec = Literal(
|
||||||
Decimal(BigDecimal(60.5), 8, 6))), EmptyRow, "The fraction of sec must be zero")
|
Decimal(BigDecimal(60.5), 8, 6))), EmptyRow, "The fraction of sec must be zero")
|
||||||
} else {
|
} else {
|
||||||
checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-07-01 00:00:00"))
|
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-07-01 00:00:00"))
|
||||||
}
|
}
|
||||||
|
|
||||||
makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12), Literal(0),
|
makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12), Literal(0),
|
||||||
Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
|
Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
|
||||||
checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
|
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-08-12 00:00:58.000001"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1183,14 +1192,17 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
|
withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
|
||||||
val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30),
|
val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30),
|
||||||
Literal(23), Literal(59), Literal(Decimal(BigDecimal(60.0), 8, 6)))
|
Literal(23), Literal(59), Literal(Decimal(BigDecimal(60.0), 8, 6)))
|
||||||
checkEvaluation(makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(60.5), 8, 6))), null)
|
checkEvaluation(makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(60.5), 8, 6))),
|
||||||
|
null)
|
||||||
}
|
}
|
||||||
|
|
||||||
Seq(true, false).foreach { ansi =>
|
Seq(true, false).foreach { ansi =>
|
||||||
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
|
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) {
|
||||||
val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12),
|
val makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(8), Literal(12),
|
||||||
Literal(0), Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
|
Literal(0), Literal(0), Literal(Decimal(BigDecimal(58.000001), 8, 6)))
|
||||||
checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
|
checkEvaluation(makeTimestampExpr, expectedAnswer("2019-08-12 00:00:58.000001"))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -257,3 +257,7 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '10-9' year to month;
|
||||||
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15' day to hour;
|
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15' day to hour;
|
||||||
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40' day to minute;
|
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40' day to minute;
|
||||||
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999' day to second;
|
select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999' day to second;
|
||||||
|
|
||||||
|
-- timestamp numeric fields constructor
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678);
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007);
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 193
|
-- Number of queries: 195
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -1642,3 +1642,20 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999'
|
||||||
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
||||||
-- !query output
|
-- !query output
|
||||||
2021-06-04 18:30:39.001001
|
2021-06-04 18:30:39.001001
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 45.678):timestamp>
|
||||||
|
-- !query output
|
||||||
|
2021-07-11 06:30:45.678
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007)
|
||||||
|
-- !query schema
|
||||||
|
struct<>
|
||||||
|
-- !query output
|
||||||
|
java.time.DateTimeException
|
||||||
|
The fraction of sec must be zero. Valid range is [0, 60].
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 193
|
-- Number of queries: 195
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -1584,3 +1584,19 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999'
|
||||||
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
||||||
-- !query output
|
-- !query output
|
||||||
2021-06-04 18:30:39.001001
|
2021-06-04 18:30:39.001001
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 45.678):timestamp>
|
||||||
|
-- !query output
|
||||||
|
2021-07-11 06:30:45.678
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 193
|
-- Number of queries: 195
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -1592,3 +1592,19 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999'
|
||||||
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
||||||
-- !query output
|
-- !query output
|
||||||
2021-06-04 18:30:39.001001
|
2021-06-04 18:30:39.001001
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 45.678):timestamp>
|
||||||
|
-- !query output
|
||||||
|
2021-07-11 06:30:45.678
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 193
|
-- Number of queries: 195
|
||||||
|
|
||||||
|
|
||||||
-- !query
|
-- !query
|
||||||
|
@ -1593,3 +1593,19 @@ select to_timestamp_ntz('2021-06-25 10:11:12') - interval '20 15:40:32.99899999'
|
||||||
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
struct<to_timestamp_ntz(2021-06-25 10:11:12) - INTERVAL '20 15:40:32.998999' DAY TO SECOND:timestamp_ntz>
|
||||||
-- !query output
|
-- !query output
|
||||||
2021-06-04 18:30:39.001001
|
2021-06-04 18:30:39.001001
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 45.678)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 45.678):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
2021-07-11 06:30:45.678
|
||||||
|
|
||||||
|
|
||||||
|
-- !query
|
||||||
|
SELECT make_timestamp(2021, 07, 11, 6, 30, 60.007)
|
||||||
|
-- !query schema
|
||||||
|
struct<make_timestamp(2021, 7, 11, 6, 30, 60.007):timestamp_ntz>
|
||||||
|
-- !query output
|
||||||
|
NULL
|
||||||
|
|
Loading…
Reference in a new issue