[SPARK-35978][SQL] Support non-reserved keyword TIMESTAMP_LTZ
### What changes were proposed in this pull request?
Support new keyword `TIMESTAMP_LTZ`, which can be used for:
- timestamp with local time zone data type in DDL
- timestamp with local time zone data type in Cast clause.
- timestamp with local time zone data type literal
### Why are the changes needed?
Users can use `TIMESTAMP_LTZ` in DDL/Cast/Literals for the timestamp with local time zone type directly. The new keyword is independent of the SQL configuration `spark.sql.timestampType`.
### Does this PR introduce _any_ user-facing change?
No, the new timestamp type is not released yet.
### How was this patch tested?
Unit test
Closes #33224 from gengliangwang/TIMESTAMP_LTZ.
Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
(cherry picked from commit b0b9643cd7
)
Signed-off-by: Gengliang Wang <gengliang@apache.org>
This commit is contained in:
parent
22b303a648
commit
e09feda1d2
|
@ -2119,6 +2119,13 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
throw QueryParsingErrors.cannotParseValueTypeError(valueType, value, ctx)
|
throw QueryParsingErrors.cannotParseValueTypeError(valueType, value, ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def constructTimestampLTZLiteral(value: String): Literal = {
|
||||||
|
val zoneId = getZoneId(conf.sessionLocalTimeZone)
|
||||||
|
val specialTs = convertSpecialTimestamp(value, zoneId).map(Literal(_, TimestampType))
|
||||||
|
specialTs.getOrElse(toLiteral(stringToTimestamp(_, zoneId), TimestampType))
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
valueType match {
|
valueType match {
|
||||||
case "DATE" =>
|
case "DATE" =>
|
||||||
|
@ -2128,13 +2135,9 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
case "TIMESTAMP_NTZ" =>
|
case "TIMESTAMP_NTZ" =>
|
||||||
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
|
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
|
||||||
specialTs.getOrElse(toLiteral(stringToTimestampWithoutTimeZone, TimestampNTZType))
|
specialTs.getOrElse(toLiteral(stringToTimestampWithoutTimeZone, TimestampNTZType))
|
||||||
|
case "TIMESTAMP_LTZ" =>
|
||||||
|
constructTimestampLTZLiteral(value)
|
||||||
case "TIMESTAMP" =>
|
case "TIMESTAMP" =>
|
||||||
def constructTimestampLTZLiteral(value: String): Literal = {
|
|
||||||
val zoneId = getZoneId(conf.sessionLocalTimeZone)
|
|
||||||
val specialTs = convertSpecialTimestamp(value, zoneId).map(Literal(_, TimestampType))
|
|
||||||
specialTs.getOrElse(toLiteral(stringToTimestamp(_, zoneId), TimestampType))
|
|
||||||
}
|
|
||||||
|
|
||||||
SQLConf.get.timestampType match {
|
SQLConf.get.timestampType match {
|
||||||
case TimestampNTZType =>
|
case TimestampNTZType =>
|
||||||
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
|
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
|
||||||
|
@ -2529,6 +2532,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
case ("date", Nil) => DateType
|
case ("date", Nil) => DateType
|
||||||
case ("timestamp", Nil) => SQLConf.get.timestampType
|
case ("timestamp", Nil) => SQLConf.get.timestampType
|
||||||
case ("timestamp_ntz", Nil) => TimestampNTZType
|
case ("timestamp_ntz", Nil) => TimestampNTZType
|
||||||
|
case ("timestamp_ltz", Nil) => TimestampType
|
||||||
case ("string", Nil) => StringType
|
case ("string", Nil) => StringType
|
||||||
case ("character" | "char", length :: Nil) => CharType(length.getText.toInt)
|
case ("character" | "char", length :: Nil) => CharType(length.getText.toInt)
|
||||||
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
|
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
|
||||||
|
|
|
@ -59,6 +59,7 @@ class DataTypeParserSuite extends SparkFunSuite with SQLHelper {
|
||||||
checkDataType("DATE", DateType)
|
checkDataType("DATE", DateType)
|
||||||
checkDataType("timestamp", TimestampType)
|
checkDataType("timestamp", TimestampType)
|
||||||
checkDataType("timestamp_ntz", TimestampNTZType)
|
checkDataType("timestamp_ntz", TimestampNTZType)
|
||||||
|
checkDataType("timestamp_ltz", TimestampType)
|
||||||
checkDataType("string", StringType)
|
checkDataType("string", StringType)
|
||||||
checkDataType("ChaR(5)", CharType(5))
|
checkDataType("ChaR(5)", CharType(5))
|
||||||
checkDataType("ChaRacter(5)", CharType(5))
|
checkDataType("ChaRacter(5)", CharType(5))
|
||||||
|
|
|
@ -455,6 +455,17 @@ class ExpressionParserSuite extends AnalysisTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
test("type constructors") {
|
test("type constructors") {
|
||||||
|
def checkTimestampNTZAndLTZ(): Unit = {
|
||||||
|
// Timestamp with local time zone
|
||||||
|
assertEqual("tImEstAmp_LTZ '2016-03-11 20:54:00.000'",
|
||||||
|
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
|
||||||
|
intercept("timestamP_LTZ '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_LTZ value")
|
||||||
|
// Timestamp without time zone
|
||||||
|
assertEqual("tImEstAmp_Ntz '2016-03-11 20:54:00.000'",
|
||||||
|
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
|
||||||
|
intercept("tImEstAmp_Ntz '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_NTZ value")
|
||||||
|
}
|
||||||
|
|
||||||
// Dates.
|
// Dates.
|
||||||
assertEqual("dAte '2016-03-11'", Literal(Date.valueOf("2016-03-11")))
|
assertEqual("dAte '2016-03-11'", Literal(Date.valueOf("2016-03-11")))
|
||||||
intercept("DAtE 'mar 11 2016'", "Cannot parse the DATE value")
|
intercept("DAtE 'mar 11 2016'", "Cannot parse the DATE value")
|
||||||
|
@ -464,10 +475,7 @@ class ExpressionParserSuite extends AnalysisTest {
|
||||||
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
|
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
|
||||||
intercept("timestamP '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP value")
|
intercept("timestamP '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP value")
|
||||||
|
|
||||||
// Timestamp without time zone
|
checkTimestampNTZAndLTZ()
|
||||||
assertEqual("tImEstAmp_Ntz '2016-03-11 20:54:00.000'",
|
|
||||||
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
|
|
||||||
intercept("tImEstAmp_Ntz '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_NTZ value")
|
|
||||||
withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> TimestampTypes.TIMESTAMP_NTZ.toString) {
|
withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> TimestampTypes.TIMESTAMP_NTZ.toString) {
|
||||||
assertEqual("tImEstAmp '2016-03-11 20:54:00.000'",
|
assertEqual("tImEstAmp '2016-03-11 20:54:00.000'",
|
||||||
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
|
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
|
||||||
|
@ -477,6 +485,9 @@ class ExpressionParserSuite extends AnalysisTest {
|
||||||
// If the timestamp string contains time zone, return a timestamp with local time zone literal
|
// If the timestamp string contains time zone, return a timestamp with local time zone literal
|
||||||
assertEqual("tImEstAmp '1970-01-01 00:00:00.000 +01:00'",
|
assertEqual("tImEstAmp '1970-01-01 00:00:00.000 +01:00'",
|
||||||
Literal(-3600000000L, TimestampType))
|
Literal(-3600000000L, TimestampType))
|
||||||
|
|
||||||
|
// The behavior of TIMESTAMP_NTZ and TIMESTAMP_LTZ is independent of SQLConf.TIMESTAMP_TYPE
|
||||||
|
checkTimestampNTZAndLTZ()
|
||||||
}
|
}
|
||||||
// Interval.
|
// Interval.
|
||||||
val intervalLiteral = Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour"))
|
val intervalLiteral = Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour"))
|
||||||
|
|
Loading…
Reference in a new issue