From 0e554d44df64d26303844c50c4c6abf2cdba3396 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Thu, 17 Jun 2021 09:51:47 +0300 Subject: [PATCH] [SPARK-35770][SQL] Parse YearMonthIntervalType from JSON ### What changes were proposed in this pull request? Parse YearMonthIntervalType from JSON. ### Why are the changes needed? This will allow to store year-month intervals as table columns into Hive external catalog. ### Does this PR introduce _any_ user-facing change? People can store year-month interval types as json string. ### How was this patch tested? Added UT. Closes #32929 from AngersZhuuuu/SPARK-35770. Authored-by: Angerszhuuuu Signed-off-by: Max Gekk --- .../main/scala/org/apache/spark/sql/types/DataType.scala | 7 +++++-- .../scala/org/apache/spark/sql/types/DataTypeSuite.scala | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala index 35fbe5ca80..1c4ad8821e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy.{ANSI, STRICT} +import org.apache.spark.sql.types.YearMonthIntervalType._ import org.apache.spark.util.Utils /** @@ -173,8 +174,10 @@ object DataType { DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType, // TODO(SPARK-35732): Parse DayTimeIntervalType from JSON DayTimeIntervalType(), - // TODO(SPARK-35770): Parse YearMonthIntervalType from JSON - YearMonthIntervalType(), TimestampWithoutTZType) + YearMonthIntervalType(YEAR, YEAR), + YearMonthIntervalType(MONTH, MONTH), + YearMonthIntervalType(YEAR, MONTH), + TimestampWithoutTZType) .map(t => t.typeName -> t).toMap } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala index 3bf411704b..d6204150e3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala @@ -256,6 +256,9 @@ class DataTypeSuite extends SparkFunSuite { checkDataTypeFromJson(VarcharType(10)) checkDataTypeFromDDL(VarcharType(11)) + + yearMonthIntervalTypes.foreach(checkDataTypeFromJson) + yearMonthIntervalTypes.foreach(checkDataTypeFromDDL) dayTimeIntervalTypes.foreach(checkDataTypeFromDDL)