ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35732][SQL] Parse DayTimeIntervalType from JSON

Browse source 
### What changes were proposed in this pull request?
Support Parse DayTimeIntervalType from JSON

### Why are the changes needed?
this will allow to store day-second intervals as table columns into Hive external catalog.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT

Closes #32930 from AngersZhuuuu/SPARK-35732.

Lead-authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Co-authored-by: AngersZhuuuu <angers.zhu@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Angerszhuuuu 2021-06-17 12:54:34 +03:00 committed by Max Gekk
parent 0c5a01a78c
commit 234163fbe0
2 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
View file

@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy.{ANSI, STRICT}
import org.apache.spark.sql.types.DayTimeIntervalType._
import org.apache.spark.sql.types.YearMonthIntervalType._
import org.apache.spark.util.Utils
@ -172,8 +173,16 @@ object DataType {
private val otherTypes = {
Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
// TODO(SPARK-35732): Parse DayTimeIntervalType from JSON
DayTimeIntervalType(),
DayTimeIntervalType(DAY, DAY),
DayTimeIntervalType(DAY, HOUR),
DayTimeIntervalType(DAY, MINUTE),
DayTimeIntervalType(DAY, SECOND),
DayTimeIntervalType(HOUR, HOUR),
DayTimeIntervalType(HOUR, MINUTE),
DayTimeIntervalType(HOUR, SECOND),
DayTimeIntervalType(MINUTE, MINUTE),
DayTimeIntervalType(MINUTE, SECOND),
DayTimeIntervalType(SECOND, SECOND),
YearMonthIntervalType(YEAR, YEAR),
YearMonthIntervalType(MONTH, MONTH),
YearMonthIntervalType(YEAR, MONTH),

2
sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
View file

@ -256,7 +256,7 @@ class DataTypeSuite extends SparkFunSuite {
checkDataTypeFromJson(VarcharType(10))
checkDataTypeFromDDL(VarcharType(11))
dayTimeIntervalTypes.foreach(checkDataTypeFromJson)
yearMonthIntervalTypes.foreach(checkDataTypeFromJson)
yearMonthIntervalTypes.foreach(checkDataTypeFromDDL)