diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala index e2f7c262e7..62835287d9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnAccessor.scala @@ -139,8 +139,8 @@ private[sql] object ColumnAccessor { case BooleanType => new BooleanColumnAccessor(buf) case ByteType => new ByteColumnAccessor(buf) case ShortType => new ShortColumnAccessor(buf) - case IntegerType | DateType => new IntColumnAccessor(buf) - case LongType | TimestampType => new LongColumnAccessor(buf) + case IntegerType | DateType | YearMonthIntervalType => new IntColumnAccessor(buf) + case LongType | TimestampType | DayTimeIntervalType => new LongColumnAccessor(buf) case FloatType => new FloatColumnAccessor(buf) case DoubleType => new DoubleColumnAccessor(buf) case StringType => new StringColumnAccessor(buf) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala index 9a1d788e7f..261e5dd6ef 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnBuilder.scala @@ -174,8 +174,8 @@ private[columnar] object ColumnBuilder { case BooleanType => new BooleanColumnBuilder case ByteType => new ByteColumnBuilder case ShortType => new ShortColumnBuilder - case IntegerType | DateType => new IntColumnBuilder - case LongType | TimestampType => new LongColumnBuilder + case IntegerType | DateType | YearMonthIntervalType => new IntColumnBuilder + case LongType | TimestampType | DayTimeIntervalType => new LongColumnBuilder case FloatType => new FloatColumnBuilder case DoubleType => new DoubleColumnBuilder case StringType => new StringColumnBuilder diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala index 63ae22d198..2f74fab270 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/ColumnType.scala @@ -817,8 +817,8 @@ private[columnar] object ColumnType { case BooleanType => BOOLEAN case ByteType => BYTE case ShortType => SHORT - case IntegerType | DateType => INT - case LongType | TimestampType => LONG + case IntegerType | DateType | YearMonthIntervalType => INT + case LongType | TimestampType | DayTimeIntervalType => LONG case FloatType => FLOAT case DoubleType => DOUBLE case StringType => STRING diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala index eb0663830d..376aa53ca8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala @@ -80,8 +80,8 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera case BooleanType => classOf[BooleanColumnAccessor].getName case ByteType => classOf[ByteColumnAccessor].getName case ShortType => classOf[ShortColumnAccessor].getName - case IntegerType | DateType => classOf[IntColumnAccessor].getName - case LongType | TimestampType => classOf[LongColumnAccessor].getName + case IntegerType | DateType | YearMonthIntervalType => classOf[IntColumnAccessor].getName + case LongType | TimestampType | DayTimeIntervalType => classOf[LongColumnAccessor].getName case FloatType => classOf[FloatColumnAccessor].getName case DoubleType => classOf[DoubleColumnAccessor].getName case StringType => classOf[StringColumnAccessor].getName diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala index a98f4d5f49..e1a2198516 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import java.io.{File, FilenameFilter} import java.nio.file.{Files, Paths} +import java.time.{Duration, Period} import scala.collection.mutable.HashSet import scala.concurrent.duration._ @@ -1187,6 +1188,20 @@ class CachedTableSuite extends QueryTest with SQLTestUtils } } + test("SPARK-35243: cache supports for YearMonthIntervalType and DayTimeIntervalType") { + withTempView("ymi_dti_interval_cache") { + Seq((1, Period.ofYears(1), Duration.ofDays(1)), + (2, Period.ofYears(2), Duration.ofDays(2))) + .toDF("k", "v1", "v2").createTempView("ymi_dti_interval_cache") + sql("CACHE TABLE tmp AS SELECT k, v1, v2 FROM ymi_dti_interval_cache") + assert(spark.catalog.isCached("tmp")) + checkAnswer(sql("SELECT * FROM tmp WHERE k = 1"), + Row(1, Period.ofYears(1), Duration.ofDays(1))) + sql("UNCACHE TABLE tmp") + assert(!spark.catalog.isCached("tmp")) + } + } + test("SPARK-30494 Fix the leak of cached data when replace an existing view") { withTempView("tempView") { spark.catalog.clearCache()