[SPARK-35978][SQL] Support non-reserved keyword TIMESTAMP_LTZ

### What changes were proposed in this pull request?

Support new keyword `TIMESTAMP_LTZ`, which can be used for:

- timestamp with local time zone data type in DDL
- timestamp with local time zone data type in Cast clause.
- timestamp with local time zone data type literal

### Why are the changes needed?

Users can use `TIMESTAMP_LTZ` in DDL/Cast/Literals for the timestamp with local time zone type directly. The new keyword is independent of the SQL configuration `spark.sql.timestampType`.

### Does this PR introduce _any_ user-facing change?

No, the new timestamp type is not released yet.

### How was this patch tested?

Unit test

Closes #33224 from gengliangwang/TIMESTAMP_LTZ.

Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
(cherry picked from commit b0b9643cd7)
Signed-off-by: Gengliang Wang <gengliang@apache.org>
This commit is contained in:
Gengliang Wang 2021-07-06 14:33:22 +08:00
parent 22b303a648
commit e09feda1d2
3 changed files with 26 additions and 10 deletions

View file

@ -2119,6 +2119,13 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
throw QueryParsingErrors.cannotParseValueTypeError(valueType, value, ctx) throw QueryParsingErrors.cannotParseValueTypeError(valueType, value, ctx)
} }
} }
def constructTimestampLTZLiteral(value: String): Literal = {
val zoneId = getZoneId(conf.sessionLocalTimeZone)
val specialTs = convertSpecialTimestamp(value, zoneId).map(Literal(_, TimestampType))
specialTs.getOrElse(toLiteral(stringToTimestamp(_, zoneId), TimestampType))
}
try { try {
valueType match { valueType match {
case "DATE" => case "DATE" =>
@ -2128,13 +2135,9 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
case "TIMESTAMP_NTZ" => case "TIMESTAMP_NTZ" =>
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType)) val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
specialTs.getOrElse(toLiteral(stringToTimestampWithoutTimeZone, TimestampNTZType)) specialTs.getOrElse(toLiteral(stringToTimestampWithoutTimeZone, TimestampNTZType))
case "TIMESTAMP_LTZ" =>
constructTimestampLTZLiteral(value)
case "TIMESTAMP" => case "TIMESTAMP" =>
def constructTimestampLTZLiteral(value: String): Literal = {
val zoneId = getZoneId(conf.sessionLocalTimeZone)
val specialTs = convertSpecialTimestamp(value, zoneId).map(Literal(_, TimestampType))
specialTs.getOrElse(toLiteral(stringToTimestamp(_, zoneId), TimestampType))
}
SQLConf.get.timestampType match { SQLConf.get.timestampType match {
case TimestampNTZType => case TimestampNTZType =>
val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType)) val specialTs = convertSpecialTimestampNTZ(value).map(Literal(_, TimestampNTZType))
@ -2529,6 +2532,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
case ("date", Nil) => DateType case ("date", Nil) => DateType
case ("timestamp", Nil) => SQLConf.get.timestampType case ("timestamp", Nil) => SQLConf.get.timestampType
case ("timestamp_ntz", Nil) => TimestampNTZType case ("timestamp_ntz", Nil) => TimestampNTZType
case ("timestamp_ltz", Nil) => TimestampType
case ("string", Nil) => StringType case ("string", Nil) => StringType
case ("character" | "char", length :: Nil) => CharType(length.getText.toInt) case ("character" | "char", length :: Nil) => CharType(length.getText.toInt)
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt) case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)

View file

@ -59,6 +59,7 @@ class DataTypeParserSuite extends SparkFunSuite with SQLHelper {
checkDataType("DATE", DateType) checkDataType("DATE", DateType)
checkDataType("timestamp", TimestampType) checkDataType("timestamp", TimestampType)
checkDataType("timestamp_ntz", TimestampNTZType) checkDataType("timestamp_ntz", TimestampNTZType)
checkDataType("timestamp_ltz", TimestampType)
checkDataType("string", StringType) checkDataType("string", StringType)
checkDataType("ChaR(5)", CharType(5)) checkDataType("ChaR(5)", CharType(5))
checkDataType("ChaRacter(5)", CharType(5)) checkDataType("ChaRacter(5)", CharType(5))

View file

@ -455,6 +455,17 @@ class ExpressionParserSuite extends AnalysisTest {
} }
test("type constructors") { test("type constructors") {
def checkTimestampNTZAndLTZ(): Unit = {
// Timestamp with local time zone
assertEqual("tImEstAmp_LTZ '2016-03-11 20:54:00.000'",
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
intercept("timestamP_LTZ '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_LTZ value")
// Timestamp without time zone
assertEqual("tImEstAmp_Ntz '2016-03-11 20:54:00.000'",
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
intercept("tImEstAmp_Ntz '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_NTZ value")
}
// Dates. // Dates.
assertEqual("dAte '2016-03-11'", Literal(Date.valueOf("2016-03-11"))) assertEqual("dAte '2016-03-11'", Literal(Date.valueOf("2016-03-11")))
intercept("DAtE 'mar 11 2016'", "Cannot parse the DATE value") intercept("DAtE 'mar 11 2016'", "Cannot parse the DATE value")
@ -464,10 +475,7 @@ class ExpressionParserSuite extends AnalysisTest {
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000"))) Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
intercept("timestamP '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP value") intercept("timestamP '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP value")
// Timestamp without time zone checkTimestampNTZAndLTZ()
assertEqual("tImEstAmp_Ntz '2016-03-11 20:54:00.000'",
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
intercept("tImEstAmp_Ntz '2016-33-11 20:54:00.000'", "Cannot parse the TIMESTAMP_NTZ value")
withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> TimestampTypes.TIMESTAMP_NTZ.toString) { withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> TimestampTypes.TIMESTAMP_NTZ.toString) {
assertEqual("tImEstAmp '2016-03-11 20:54:00.000'", assertEqual("tImEstAmp '2016-03-11 20:54:00.000'",
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000"))) Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
@ -477,6 +485,9 @@ class ExpressionParserSuite extends AnalysisTest {
// If the timestamp string contains time zone, return a timestamp with local time zone literal // If the timestamp string contains time zone, return a timestamp with local time zone literal
assertEqual("tImEstAmp '1970-01-01 00:00:00.000 +01:00'", assertEqual("tImEstAmp '1970-01-01 00:00:00.000 +01:00'",
Literal(-3600000000L, TimestampType)) Literal(-3600000000L, TimestampType))
// The behavior of TIMESTAMP_NTZ and TIMESTAMP_LTZ is independent of SQLConf.TIMESTAMP_TYPE
checkTimestampNTZAndLTZ()
} }
// Interval. // Interval.
val intervalLiteral = Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour")) val intervalLiteral = Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour"))