[SPARK-35680][SQL] Add fields to YearMonthIntervalType

### What changes were proposed in this pull request?
Extend `YearMonthIntervalType` to support interval fields. Valid interval field values:
- 0 (YEAR)
- 1 (MONTH)

After the changes, the following year-month interval types are supported:
1. `YearMonthIntervalType(0, 0)` or `YearMonthIntervalType(YEAR, YEAR)`
2. `YearMonthIntervalType(0, 1)` or `YearMonthIntervalType(YEAR, MONTH)`. **It is the default one**.
3. `YearMonthIntervalType(1, 1)` or `YearMonthIntervalType(MONTH, MONTH)`

Closes #32825

### Why are the changes needed?
In the current implementation, Spark supports only `interval year to month` but the SQL standard allows to specify the start and end fields. The changes will allow to follow ANSI SQL standard more precisely.

### Does this PR introduce _any_ user-facing change?
Yes but `YearMonthIntervalType` has not been released yet.

### How was this patch tested?
By existing test suites.

Closes #32909 from MaxGekk/add-fields-to-YearMonthIntervalType.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Max Gekk 2021-06-15 23:08:12 +03:00
parent 1012967ace
commit 61ce8f7649
66 changed files with 340 additions and 220 deletions

View file

@ -76,7 +76,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
*/ */
public static final Set<DataType> mutableFieldTypes; public static final Set<DataType> mutableFieldTypes;
// DecimalType and DayTimeIntervalType are also mutable // DecimalType, DayTimeIntervalType and YearMonthIntervalType are also mutable
static { static {
mutableFieldTypes = Collections.unmodifiableSet( mutableFieldTypes = Collections.unmodifiableSet(
new HashSet<>( new HashSet<>(
@ -90,8 +90,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
FloatType, FloatType,
DoubleType, DoubleType,
DateType, DateType,
TimestampType, TimestampType
YearMonthIntervalType
}))); })));
} }
@ -103,7 +102,8 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
if (dt instanceof DecimalType) { if (dt instanceof DecimalType) {
return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS(); return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
} else { } else {
return dt instanceof DayTimeIntervalType || mutableFieldTypes.contains(dt); return dt instanceof DayTimeIntervalType || dt instanceof YearMonthIntervalType ||
mutableFieldTypes.contains(dt);
} }
} }
@ -113,7 +113,8 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
} }
return mutableFieldTypes.contains(dt) || dt instanceof DecimalType || return mutableFieldTypes.contains(dt) || dt instanceof DecimalType ||
dt instanceof CalendarIntervalType || dt instanceof DayTimeIntervalType; dt instanceof CalendarIntervalType || dt instanceof DayTimeIntervalType ||
dt instanceof YearMonthIntervalType;
} }
////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////

View file

@ -99,11 +99,6 @@ public class DataTypes {
*/ */
public static final DataType NullType = NullType$.MODULE$; public static final DataType NullType = NullType$.MODULE$;
/**
* Gets the YearMonthIntervalType object.
*/
public static final DataType YearMonthIntervalType = YearMonthIntervalType$.MODULE$;
/** /**
* Creates an ArrayType by specifying the data type of elements ({@code elementType}). * Creates an ArrayType by specifying the data type of elements ({@code elementType}).
* The field of {@code containsNull} is set to {@code true}. * The field of {@code containsNull} is set to {@code true}.
@ -154,6 +149,20 @@ public class DataTypes {
return DayTimeIntervalType$.MODULE$.DEFAULT(); return DayTimeIntervalType$.MODULE$.DEFAULT();
} }
/**
* Creates a YearMonthIntervalType by specifying the start and end fields.
*/
public static YearMonthIntervalType createYearMonthIntervalType(byte startField, byte endField) {
return YearMonthIntervalType$.MODULE$.apply(startField, endField);
}
/**
* Creates a YearMonthIntervalType with default start and end fields: interval year to month.
*/
public static YearMonthIntervalType createYearMonthIntervalType() {
return YearMonthIntervalType$.MODULE$.DEFAULT();
}
/** /**
* Creates a MapType by specifying the data type of keys ({@code keyType}) and values * Creates a MapType by specifying the data type of keys ({@code keyType}) and values
* ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}. * ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}.

View file

@ -77,7 +77,8 @@ object CatalystTypeConverters {
case DoubleType => DoubleConverter case DoubleType => DoubleConverter
// TODO(SPARK-35726): Truncate java.time.Duration by fields of day-time interval type // TODO(SPARK-35726): Truncate java.time.Duration by fields of day-time interval type
case _: DayTimeIntervalType => DurationConverter case _: DayTimeIntervalType => DurationConverter
case YearMonthIntervalType => PeriodConverter // TODO(SPARK-35769): Truncate java.time.Period by fields of year-month interval type
case _: YearMonthIntervalType => PeriodConverter
case dataType: DataType => IdentityConverter(dataType) case dataType: DataType => IdentityConverter(dataType)
} }
converter.asInstanceOf[CatalystTypeConverter[Any, Any, Any]] converter.asInstanceOf[CatalystTypeConverter[Any, Any, Any]]

View file

@ -132,7 +132,7 @@ object InternalRow {
case BooleanType => (input, ordinal) => input.getBoolean(ordinal) case BooleanType => (input, ordinal) => input.getBoolean(ordinal)
case ByteType => (input, ordinal) => input.getByte(ordinal) case ByteType => (input, ordinal) => input.getByte(ordinal)
case ShortType => (input, ordinal) => input.getShort(ordinal) case ShortType => (input, ordinal) => input.getShort(ordinal)
case IntegerType | DateType | YearMonthIntervalType => case IntegerType | DateType | _: YearMonthIntervalType =>
(input, ordinal) => input.getInt(ordinal) (input, ordinal) => input.getInt(ordinal)
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
(input, ordinal) => input.getLong(ordinal) (input, ordinal) => input.getLong(ordinal)
@ -169,7 +169,7 @@ object InternalRow {
case BooleanType => (input, v) => input.setBoolean(ordinal, v.asInstanceOf[Boolean]) case BooleanType => (input, v) => input.setBoolean(ordinal, v.asInstanceOf[Boolean])
case ByteType => (input, v) => input.setByte(ordinal, v.asInstanceOf[Byte]) case ByteType => (input, v) => input.setByte(ordinal, v.asInstanceOf[Byte])
case ShortType => (input, v) => input.setShort(ordinal, v.asInstanceOf[Short]) case ShortType => (input, v) => input.setShort(ordinal, v.asInstanceOf[Short])
case IntegerType | DateType | YearMonthIntervalType => case IntegerType | DateType | _: YearMonthIntervalType =>
(input, v) => input.setInt(ordinal, v.asInstanceOf[Int]) (input, v) => input.setInt(ordinal, v.asInstanceOf[Int])
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
(input, v) => input.setLong(ordinal, v.asInstanceOf[Long]) (input, v) => input.setLong(ordinal, v.asInstanceOf[Long])

View file

@ -121,7 +121,7 @@ object JavaTypeInference {
case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true) case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true)
case c: Class[_] if c == classOf[java.time.LocalDateTime] => (TimestampWithoutTZType, true) case c: Class[_] if c == classOf[java.time.LocalDateTime] => (TimestampWithoutTZType, true)
case c: Class[_] if c == classOf[java.time.Duration] => (DayTimeIntervalType(), true) case c: Class[_] if c == classOf[java.time.Duration] => (DayTimeIntervalType(), true)
case c: Class[_] if c == classOf[java.time.Period] => (YearMonthIntervalType, true) case c: Class[_] if c == classOf[java.time.Period] => (YearMonthIntervalType(), true)
case _ if typeToken.isArray => case _ if typeToken.isArray =>
val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet) val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet)

View file

@ -761,7 +761,7 @@ object ScalaReflection extends ScalaReflection {
case t if isSubtype(t, localTypeOf[java.time.Duration]) => case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
Schema(DayTimeIntervalType(), nullable = true) Schema(DayTimeIntervalType(), nullable = true)
case t if isSubtype(t, localTypeOf[java.time.Period]) => case t if isSubtype(t, localTypeOf[java.time.Period]) =>
Schema(YearMonthIntervalType, nullable = true) Schema(YearMonthIntervalType(), nullable = true)
case t if isSubtype(t, localTypeOf[BigDecimal]) => case t if isSubtype(t, localTypeOf[BigDecimal]) =>
Schema(DecimalType.SYSTEM_DEFAULT, nullable = true) Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) => case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
@ -860,8 +860,7 @@ object ScalaReflection extends ScalaReflection {
TimestampType -> classOf[TimestampType.InternalType], TimestampType -> classOf[TimestampType.InternalType],
TimestampWithoutTZType -> classOf[TimestampWithoutTZType.InternalType], TimestampWithoutTZType -> classOf[TimestampWithoutTZType.InternalType],
BinaryType -> classOf[BinaryType.InternalType], BinaryType -> classOf[BinaryType.InternalType],
CalendarIntervalType -> classOf[CalendarInterval], CalendarIntervalType -> classOf[CalendarInterval]
YearMonthIntervalType -> classOf[YearMonthIntervalType.InternalType]
) )
val typeBoxedJavaMapping = Map[DataType, Class[_]]( val typeBoxedJavaMapping = Map[DataType, Class[_]](
@ -874,14 +873,14 @@ object ScalaReflection extends ScalaReflection {
DoubleType -> classOf[java.lang.Double], DoubleType -> classOf[java.lang.Double],
DateType -> classOf[java.lang.Integer], DateType -> classOf[java.lang.Integer],
TimestampType -> classOf[java.lang.Long], TimestampType -> classOf[java.lang.Long],
TimestampWithoutTZType -> classOf[java.lang.Long], TimestampWithoutTZType -> classOf[java.lang.Long]
YearMonthIntervalType -> classOf[java.lang.Integer]
) )
def dataTypeJavaClass(dt: DataType): Class[_] = { def dataTypeJavaClass(dt: DataType): Class[_] = {
dt match { dt match {
case _: DecimalType => classOf[Decimal] case _: DecimalType => classOf[Decimal]
case it: DayTimeIntervalType => classOf[it.InternalType] case it: DayTimeIntervalType => classOf[it.InternalType]
case it: YearMonthIntervalType => classOf[it.InternalType]
case _: StructType => classOf[InternalRow] case _: StructType => classOf[InternalRow]
case _: ArrayType => classOf[ArrayData] case _: ArrayType => classOf[ArrayData]
case _: MapType => classOf[MapData] case _: MapType => classOf[MapData]
@ -893,6 +892,7 @@ object ScalaReflection extends ScalaReflection {
def javaBoxedType(dt: DataType): Class[_] = dt match { def javaBoxedType(dt: DataType): Class[_] = dt match {
case _: DecimalType => classOf[Decimal] case _: DecimalType => classOf[Decimal]
case _: DayTimeIntervalType => classOf[java.lang.Long] case _: DayTimeIntervalType => classOf[java.lang.Long]
case _: YearMonthIntervalType => classOf[java.lang.Integer]
case BinaryType => classOf[Array[Byte]] case BinaryType => classOf[Array[Byte]]
case StringType => classOf[UTF8String] case StringType => classOf[UTF8String]
case CalendarIntervalType => classOf[CalendarInterval] case CalendarIntervalType => classOf[CalendarInterval]

View file

@ -125,7 +125,7 @@ object SerializerBuildHelper {
def createSerializerForJavaPeriod(inputObject: Expression): Expression = { def createSerializerForJavaPeriod(inputObject: Expression): Expression = {
StaticInvoke( StaticInvoke(
IntervalUtils.getClass, IntervalUtils.getClass,
YearMonthIntervalType, YearMonthIntervalType(),
"periodToMonths", "periodToMonths",
inputObject :: Nil, inputObject :: Nil,
returnNullable = false) returnNullable = false)

View file

@ -351,10 +351,10 @@ class Analyzer(override val catalogManager: CatalogManager)
case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match { case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match {
case (DateType, _: DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r) case (DateType, _: DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r)
case (_: DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l) case (_: DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l)
case (DateType, YearMonthIntervalType) => DateAddYMInterval(l, r) case (DateType, _: YearMonthIntervalType) => DateAddYMInterval(l, r)
case (YearMonthIntervalType, DateType) => DateAddYMInterval(r, l) case (_: YearMonthIntervalType, DateType) => DateAddYMInterval(r, l)
case (TimestampType, YearMonthIntervalType) => TimestampAddYMInterval(l, r) case (TimestampType, _: YearMonthIntervalType) => TimestampAddYMInterval(l, r)
case (YearMonthIntervalType, TimestampType) => TimestampAddYMInterval(r, l) case (_: YearMonthIntervalType, TimestampType) => TimestampAddYMInterval(r, l)
case (CalendarIntervalType, CalendarIntervalType) | case (CalendarIntervalType, CalendarIntervalType) |
(_: DayTimeIntervalType, _: DayTimeIntervalType) => a (_: DayTimeIntervalType, _: DayTimeIntervalType) => a
case (DateType, CalendarIntervalType) => DateAddInterval(l, r, ansiEnabled = f) case (DateType, CalendarIntervalType) => DateAddInterval(l, r, ansiEnabled = f)
@ -368,9 +368,9 @@ class Analyzer(override val catalogManager: CatalogManager)
case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match { case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match {
case (DateType, _: DayTimeIntervalType) => case (DateType, _: DayTimeIntervalType) =>
DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f))) DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f)))
case (DateType, YearMonthIntervalType) => case (DateType, _: YearMonthIntervalType) =>
DatetimeSub(l, r, DateAddYMInterval(l, UnaryMinus(r, f))) DatetimeSub(l, r, DateAddYMInterval(l, UnaryMinus(r, f)))
case (TimestampType, YearMonthIntervalType) => case (TimestampType, _: YearMonthIntervalType) =>
DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, f))) DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, f)))
case (CalendarIntervalType, CalendarIntervalType) | case (CalendarIntervalType, CalendarIntervalType) |
(_: DayTimeIntervalType, _: DayTimeIntervalType) => s (_: DayTimeIntervalType, _: DayTimeIntervalType) => s
@ -387,15 +387,15 @@ class Analyzer(override val catalogManager: CatalogManager)
case m @ Multiply(l, r, f) if m.childrenResolved => (l.dataType, r.dataType) match { case m @ Multiply(l, r, f) if m.childrenResolved => (l.dataType, r.dataType) match {
case (CalendarIntervalType, _) => MultiplyInterval(l, r, f) case (CalendarIntervalType, _) => MultiplyInterval(l, r, f)
case (_, CalendarIntervalType) => MultiplyInterval(r, l, f) case (_, CalendarIntervalType) => MultiplyInterval(r, l, f)
case (YearMonthIntervalType, _) => MultiplyYMInterval(l, r) case (_: YearMonthIntervalType, _) => MultiplyYMInterval(l, r)
case (_, YearMonthIntervalType) => MultiplyYMInterval(r, l) case (_, _: YearMonthIntervalType) => MultiplyYMInterval(r, l)
case (_: DayTimeIntervalType, _) => MultiplyDTInterval(l, r) case (_: DayTimeIntervalType, _) => MultiplyDTInterval(l, r)
case (_, _: DayTimeIntervalType) => MultiplyDTInterval(r, l) case (_, _: DayTimeIntervalType) => MultiplyDTInterval(r, l)
case _ => m case _ => m
} }
case d @ Divide(l, r, f) if d.childrenResolved => (l.dataType, r.dataType) match { case d @ Divide(l, r, f) if d.childrenResolved => (l.dataType, r.dataType) match {
case (CalendarIntervalType, _) => DivideInterval(l, r, f) case (CalendarIntervalType, _) => DivideInterval(l, r, f)
case (YearMonthIntervalType, _) => DivideYMInterval(l, r) case (_: YearMonthIntervalType, _) => DivideYMInterval(l, r)
case (_: DayTimeIntervalType, _) => DivideDTInterval(l, r) case (_: DayTimeIntervalType, _) => DivideDTInterval(l, r)
case _ => d case _ => d
} }

View file

@ -310,8 +310,11 @@ package object dsl {
} }
/** Creates a new AttributeReference of the year-month interval type */ /** Creates a new AttributeReference of the year-month interval type */
def yearMonthInterval: AttributeReference = { def yearMonthInterval(startField: Byte, endField: Byte): AttributeReference = {
AttributeReference(s, YearMonthIntervalType, nullable = true)() AttributeReference(s, YearMonthIntervalType(startField, endField), nullable = true)()
}
def yearMonthInterval(): AttributeReference = {
AttributeReference(s, YearMonthIntervalType(), nullable = true)()
} }
/** Creates a new AttributeReference of type binary */ /** Creates a new AttributeReference of type binary */

View file

@ -116,7 +116,7 @@ object RowEncoder {
case _: DayTimeIntervalType => createSerializerForJavaDuration(inputObject) case _: DayTimeIntervalType => createSerializerForJavaDuration(inputObject)
case YearMonthIntervalType => createSerializerForJavaPeriod(inputObject) case _: YearMonthIntervalType => createSerializerForJavaPeriod(inputObject)
case d: DecimalType => case d: DecimalType =>
CheckOverflow(StaticInvoke( CheckOverflow(StaticInvoke(
@ -239,7 +239,7 @@ object RowEncoder {
ObjectType(classOf[java.sql.Date]) ObjectType(classOf[java.sql.Date])
} }
case _: DayTimeIntervalType => ObjectType(classOf[java.time.Duration]) case _: DayTimeIntervalType => ObjectType(classOf[java.time.Duration])
case YearMonthIntervalType => ObjectType(classOf[java.time.Period]) case _: YearMonthIntervalType => ObjectType(classOf[java.time.Period])
case _: DecimalType => ObjectType(classOf[java.math.BigDecimal]) case _: DecimalType => ObjectType(classOf[java.math.BigDecimal])
case StringType => ObjectType(classOf[java.lang.String]) case StringType => ObjectType(classOf[java.lang.String])
case _: ArrayType => ObjectType(classOf[scala.collection.Seq[_]]) case _: ArrayType => ObjectType(classOf[scala.collection.Seq[_]])
@ -299,7 +299,7 @@ object RowEncoder {
case _: DayTimeIntervalType => createDeserializerForDuration(input) case _: DayTimeIntervalType => createDeserializerForDuration(input)
case YearMonthIntervalType => createDeserializerForPeriod(input) case _: YearMonthIntervalType => createDeserializerForPeriod(input)
case _: DecimalType => createDeserializerForJavaBigDecimal(input, returnNullable = false) case _: DecimalType => createDeserializerForJavaBigDecimal(input, returnNullable = false)

View file

@ -79,7 +79,7 @@ object Cast {
case (StringType, CalendarIntervalType) => true case (StringType, CalendarIntervalType) => true
case (StringType, _: DayTimeIntervalType) => true case (StringType, _: DayTimeIntervalType) => true
case (StringType, YearMonthIntervalType) => true case (StringType, _: YearMonthIntervalType) => true
case (StringType, _: NumericType) => true case (StringType, _: NumericType) => true
case (BooleanType, _: NumericType) => true case (BooleanType, _: NumericType) => true
@ -422,9 +422,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
case pudt: PythonUserDefinedType => castToString(pudt.sqlType) case pudt: PythonUserDefinedType => castToString(pudt.sqlType)
case udt: UserDefinedType[_] => case udt: UserDefinedType[_] =>
buildCast[Any](_, o => UTF8String.fromString(udt.deserialize(o).toString)) buildCast[Any](_, o => UTF8String.fromString(udt.deserialize(o).toString))
case YearMonthIntervalType => case YearMonthIntervalType(startField, endField) =>
buildCast[Int](_, i => UTF8String.fromString( buildCast[Int](_, i => UTF8String.fromString(
IntervalUtils.toYearMonthIntervalString(i, ANSI_STYLE))) IntervalUtils.toYearMonthIntervalString(i, ANSI_STYLE, startField, endField)))
case DayTimeIntervalType(startField, endField) => case DayTimeIntervalType(startField, endField) =>
buildCast[Long](_, i => UTF8String.fromString( buildCast[Long](_, i => UTF8String.fromString(
IntervalUtils.toDayTimeIntervalString(i, ANSI_STYLE, startField, endField))) IntervalUtils.toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)))
@ -566,8 +566,11 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
IntervalUtils.castStringToDTInterval(s, it.startField, it.endField)) IntervalUtils.castStringToDTInterval(s, it.startField, it.endField))
} }
private[this] def castToYearMonthInterval(from: DataType): Any => Any = from match { private[this] def castToYearMonthInterval(
case StringType => buildCast[UTF8String](_, s => IntervalUtils.castStringToYMInterval(s)) from: DataType,
it: YearMonthIntervalType): Any => Any = from match {
case StringType => buildCast[UTF8String](_, s =>
IntervalUtils.castStringToYMInterval(s, it.startField, it.endField))
} }
// LongConverter // LongConverter
@ -876,7 +879,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
case TimestampWithoutTZType => castToTimestampWithoutTZ(from) case TimestampWithoutTZType => castToTimestampWithoutTZ(from)
case CalendarIntervalType => castToInterval(from) case CalendarIntervalType => castToInterval(from)
case it: DayTimeIntervalType => castToDayTimeInterval(from, it) case it: DayTimeIntervalType => castToDayTimeInterval(from, it)
case YearMonthIntervalType => castToYearMonthInterval(from) case it: YearMonthIntervalType => castToYearMonthInterval(from, it)
case BooleanType => castToBoolean(from) case BooleanType => castToBoolean(from)
case ByteType => castToByte(from) case ByteType => castToByte(from)
case ShortType => castToShort(from) case ShortType => castToShort(from)
@ -937,7 +940,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
case TimestampWithoutTZType => castToTimestampWithoutTZCode(from, ctx) case TimestampWithoutTZType => castToTimestampWithoutTZCode(from, ctx)
case CalendarIntervalType => castToIntervalCode(from) case CalendarIntervalType => castToIntervalCode(from)
case it: DayTimeIntervalType => castToDayTimeIntervalCode(from, it) case it: DayTimeIntervalType => castToDayTimeIntervalCode(from, it)
case YearMonthIntervalType => castToYearMonthIntervalCode(from) case it: YearMonthIntervalType => castToYearMonthIntervalCode(from, it)
case BooleanType => castToBooleanCode(from) case BooleanType => castToBooleanCode(from)
case ByteType => castToByteCode(from, ctx) case ByteType => castToByteCode(from, ctx)
case ShortType => castToShortCode(from, ctx) case ShortType => castToShortCode(from, ctx)
@ -1176,15 +1179,16 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
(c, evPrim, evNull) => { (c, evPrim, evNull) => {
code"$evPrim = UTF8String.fromString($udtRef.deserialize($c).toString());" code"$evPrim = UTF8String.fromString($udtRef.deserialize($c).toString());"
} }
case YearMonthIntervalType => case i: YearMonthIntervalType =>
val iu = IntervalUtils.getClass.getName.stripSuffix("$") val iu = IntervalUtils.getClass.getName.stripSuffix("$")
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$") val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
val style = s"$iss$$.MODULE$$.ANSI_STYLE()" val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
(c, evPrim, _) => (c, evPrim, _) =>
code""" code"""
$evPrim = UTF8String.fromString($iu.toYearMonthIntervalString($c, $style)); $evPrim = UTF8String.fromString($iu.toYearMonthIntervalString($c, $style,
(byte)${i.startField}, (byte)${i.endField}));
""" """
case i : DayTimeIntervalType => case i: DayTimeIntervalType =>
val iu = IntervalUtils.getClass.getName.stripSuffix("$") val iu = IntervalUtils.getClass.getName.stripSuffix("$")
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$") val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
val style = s"$iss$$.MODULE$$.ANSI_STYLE()" val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
@ -1441,10 +1445,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
""" """
} }
private[this] def castToYearMonthIntervalCode(from: DataType): CastFunction = from match { private[this] def castToYearMonthIntervalCode(
from: DataType,
it: YearMonthIntervalType): CastFunction = from match {
case StringType => case StringType =>
val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$") val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
(c, evPrim, _) => code"$evPrim = $util.castStringToYMInterval($c);" (c, evPrim, _) =>
code"""
$evPrim = $util.castStringToYMInterval($c, (byte)${it.startField}, (byte)${it.endField});
"""
} }
private[this] def decimalToTimestampCode(d: ExprValue): Block = { private[this] def decimalToTimestampCode(d: ExprValue): Block = {
@ -2012,7 +2021,7 @@ object AnsiCast {
case (StringType, _: CalendarIntervalType) => true case (StringType, _: CalendarIntervalType) => true
case (StringType, _: DayTimeIntervalType) => true case (StringType, _: DayTimeIntervalType) => true
case (StringType, YearMonthIntervalType) => true case (StringType, _: YearMonthIntervalType) => true
case (StringType, DateType) => true case (StringType, DateType) => true
case (TimestampType, DateType) => true case (TimestampType, DateType) => true

View file

@ -157,7 +157,7 @@ object InterpretedUnsafeProjection {
case ShortType => case ShortType =>
(v, i) => writer.write(i, v.getShort(i)) (v, i) => writer.write(i, v.getShort(i))
case IntegerType | DateType | YearMonthIntervalType => case IntegerType | DateType | _: YearMonthIntervalType =>
(v, i) => writer.write(i, v.getInt(i)) (v, i) => writer.write(i, v.getInt(i))
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>

View file

@ -194,7 +194,7 @@ final class SpecificInternalRow(val values: Array[MutableValue]) extends BaseGen
private[this] def dataTypeToMutableValue(dataType: DataType): MutableValue = dataType match { private[this] def dataTypeToMutableValue(dataType: DataType): MutableValue = dataType match {
// We use INT for DATE and YearMonthIntervalType internally // We use INT for DATE and YearMonthIntervalType internally
case IntegerType | DateType | YearMonthIntervalType => new MutableInt case IntegerType | DateType | _: YearMonthIntervalType => new MutableInt
// We use Long for Timestamp, Timestamp without time zone and DayTimeInterval internally // We use Long for Timestamp, Timestamp without time zone and DayTimeInterval internally
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
new MutableLong new MutableLong

View file

@ -57,14 +57,14 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
private lazy val resultType = child.dataType match { private lazy val resultType = child.dataType match {
case DecimalType.Fixed(p, s) => case DecimalType.Fixed(p, s) =>
DecimalType.bounded(p + 4, s + 4) DecimalType.bounded(p + 4, s + 4)
case _: YearMonthIntervalType => YearMonthIntervalType case _: YearMonthIntervalType => YearMonthIntervalType()
case _: DayTimeIntervalType => DayTimeIntervalType() case _: DayTimeIntervalType => DayTimeIntervalType()
case _ => DoubleType case _ => DoubleType
} }
private lazy val sumDataType = child.dataType match { private lazy val sumDataType = child.dataType match {
case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s) case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
case _: YearMonthIntervalType => YearMonthIntervalType case _: YearMonthIntervalType => YearMonthIntervalType()
case _: DayTimeIntervalType => DayTimeIntervalType() case _: DayTimeIntervalType => DayTimeIntervalType()
case _ => DoubleType case _ => DoubleType
} }
@ -92,7 +92,7 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
Divide(sum, count.cast(DecimalType.LongDecimal), failOnError = false)).cast(resultType) Divide(sum, count.cast(DecimalType.LongDecimal), failOnError = false)).cast(resultType)
case _: YearMonthIntervalType => case _: YearMonthIntervalType =>
If(EqualTo(count, Literal(0L)), If(EqualTo(count, Literal(0L)),
Literal(null, YearMonthIntervalType), DivideYMInterval(sum, count)) Literal(null, YearMonthIntervalType()), DivideYMInterval(sum, count))
case _: DayTimeIntervalType => case _: DayTimeIntervalType =>
If(EqualTo(count, Literal(0L)), If(EqualTo(count, Literal(0L)),
Literal(null, DayTimeIntervalType()), DivideDTInterval(sum, count)) Literal(null, DayTimeIntervalType()), DivideDTInterval(sum, count))

View file

@ -59,7 +59,7 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
case DecimalType.Fixed(precision, scale) => case DecimalType.Fixed(precision, scale) =>
DecimalType.bounded(precision + 10, scale) DecimalType.bounded(precision + 10, scale)
case _: IntegralType => LongType case _: IntegralType => LongType
case _: YearMonthIntervalType => YearMonthIntervalType case it: YearMonthIntervalType => it
case it: DayTimeIntervalType => it case it: DayTimeIntervalType => it
case _ => DoubleType case _ => DoubleType
} }

View file

@ -85,7 +85,7 @@ case class UnaryMinus(
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$") val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
val method = if (failOnError) "negateExact" else "negate" val method = if (failOnError) "negateExact" else "negate"
defineCodeGen(ctx, ev, c => s"$iu.$method($c)") defineCodeGen(ctx, ev, c => s"$iu.$method($c)")
case _: DayTimeIntervalType | YearMonthIntervalType => case _: DayTimeIntervalType | _: YearMonthIntervalType =>
nullSafeCodeGen(ctx, ev, eval => { nullSafeCodeGen(ctx, ev, eval => {
val mathClass = classOf[Math].getName val mathClass = classOf[Math].getName
s"${ev.value} = $mathClass.negateExact($eval);" s"${ev.value} = $mathClass.negateExact($eval);"
@ -97,7 +97,7 @@ case class UnaryMinus(
IntervalUtils.negateExact(input.asInstanceOf[CalendarInterval]) IntervalUtils.negateExact(input.asInstanceOf[CalendarInterval])
case CalendarIntervalType => IntervalUtils.negate(input.asInstanceOf[CalendarInterval]) case CalendarIntervalType => IntervalUtils.negate(input.asInstanceOf[CalendarInterval])
case _: DayTimeIntervalType => Math.negateExact(input.asInstanceOf[Long]) case _: DayTimeIntervalType => Math.negateExact(input.asInstanceOf[Long])
case YearMonthIntervalType => Math.negateExact(input.asInstanceOf[Int]) case _: YearMonthIntervalType => Math.negateExact(input.asInstanceOf[Int])
case _ => numeric.negate(input) case _ => numeric.negate(input)
} }
@ -229,7 +229,7 @@ abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
case CalendarIntervalType => case CalendarIntervalType =>
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$") val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
defineCodeGen(ctx, ev, (eval1, eval2) => s"$iu.$calendarIntervalMethod($eval1, $eval2)") defineCodeGen(ctx, ev, (eval1, eval2) => s"$iu.$calendarIntervalMethod($eval1, $eval2)")
case _: DayTimeIntervalType | YearMonthIntervalType => case _: DayTimeIntervalType | _: YearMonthIntervalType =>
assert(exactMathMethod.isDefined, assert(exactMathMethod.isDefined,
s"The expression '$nodeName' must override the exactMathMethod() method " + s"The expression '$nodeName' must override the exactMathMethod() method " +
"if it is supposed to operate over interval types.") "if it is supposed to operate over interval types.")
@ -319,7 +319,7 @@ case class Add(
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval]) input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
case _: DayTimeIntervalType => case _: DayTimeIntervalType =>
Math.addExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long]) Math.addExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
case YearMonthIntervalType => case _: YearMonthIntervalType =>
Math.addExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int]) Math.addExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
case _ => numeric.plus(input1, input2) case _ => numeric.plus(input1, input2)
} }
@ -365,7 +365,7 @@ case class Subtract(
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval]) input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
case _: DayTimeIntervalType => case _: DayTimeIntervalType =>
Math.subtractExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long]) Math.subtractExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
case YearMonthIntervalType => case _: YearMonthIntervalType =>
Math.subtractExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int]) Math.subtractExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
case _ => numeric.minus(input1, input2) case _ => numeric.minus(input1, input2)
} }

View file

@ -1816,7 +1816,7 @@ object CodeGenerator extends Logging {
case BooleanType => JAVA_BOOLEAN case BooleanType => JAVA_BOOLEAN
case ByteType => JAVA_BYTE case ByteType => JAVA_BYTE
case ShortType => JAVA_SHORT case ShortType => JAVA_SHORT
case IntegerType | DateType | YearMonthIntervalType => JAVA_INT case IntegerType | DateType | _: YearMonthIntervalType => JAVA_INT
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => JAVA_LONG case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => JAVA_LONG
case FloatType => JAVA_FLOAT case FloatType => JAVA_FLOAT
case DoubleType => JAVA_DOUBLE case DoubleType => JAVA_DOUBLE
@ -1837,7 +1837,7 @@ object CodeGenerator extends Logging {
case BooleanType => java.lang.Boolean.TYPE case BooleanType => java.lang.Boolean.TYPE
case ByteType => java.lang.Byte.TYPE case ByteType => java.lang.Byte.TYPE
case ShortType => java.lang.Short.TYPE case ShortType => java.lang.Short.TYPE
case IntegerType | DateType | YearMonthIntervalType => java.lang.Integer.TYPE case IntegerType | DateType | _: YearMonthIntervalType => java.lang.Integer.TYPE
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
java.lang.Long.TYPE java.lang.Long.TYPE
case FloatType => java.lang.Float.TYPE case FloatType => java.lang.Float.TYPE

View file

@ -2564,14 +2564,14 @@ case class Sequence(
|1. The start and stop expressions must resolve to the same type. |1. The start and stop expressions must resolve to the same type.
|2. If start and stop expressions resolve to the 'date' or 'timestamp' type |2. If start and stop expressions resolve to the 'date' or 'timestamp' type
|then the step expression must resolve to the 'interval' or |then the step expression must resolve to the 'interval' or
|'${YearMonthIntervalType.typeName}' or '${DayTimeIntervalType.simpleString}' type, |'${YearMonthIntervalType.simpleString}' or '${DayTimeIntervalType.simpleString}' type,
|otherwise to the same type as the start and stop expressions. |otherwise to the same type as the start and stop expressions.
""".stripMargin) """.stripMargin)
} }
} }
private def isNotIntervalType(expr: Expression) = expr.dataType match { private def isNotIntervalType(expr: Expression) = expr.dataType match {
case CalendarIntervalType | YearMonthIntervalType | _: DayTimeIntervalType => false case CalendarIntervalType | _: YearMonthIntervalType | _: DayTimeIntervalType => false
case _ => true case _ => true
} }
@ -2749,10 +2749,10 @@ object Sequence {
override val defaultStep: DefaultStep = new DefaultStep( override val defaultStep: DefaultStep = new DefaultStep(
(dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn], (dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
YearMonthIntervalType, YearMonthIntervalType(),
Period.of(0, 1, 0)) Period.of(0, 1, 0))
val intervalType: DataType = YearMonthIntervalType val intervalType: DataType = YearMonthIntervalType()
def splitStep(input: Any): (Int, Int, Long) = { def splitStep(input: Any): (Int, Int, Long) = {
(input.asInstanceOf[Int], 0, 0) (input.asInstanceOf[Int], 0, 0)

View file

@ -2396,7 +2396,7 @@ object DatePart {
throw QueryCompilationErrors.literalTypeUnsupportedForSourceTypeError(fieldStr, source) throw QueryCompilationErrors.literalTypeUnsupportedForSourceTypeError(fieldStr, source)
source.dataType match { source.dataType match {
case YearMonthIntervalType | _: DayTimeIntervalType | CalendarIntervalType => case _: YearMonthIntervalType | _: DayTimeIntervalType | CalendarIntervalType =>
ExtractIntervalPart.parseExtractField(fieldStr, source, analysisException) ExtractIntervalPart.parseExtractField(fieldStr, source, analysisException)
case _ => case _ =>
DatePart.parseExtractField(fieldStr, source, analysisException) DatePart.parseExtractField(fieldStr, source, analysisException)

View file

@ -496,7 +496,7 @@ abstract class HashExpression[E] extends Expression {
case d: DecimalType => genHashDecimal(ctx, d, input, result) case d: DecimalType => genHashDecimal(ctx, d, input, result)
case CalendarIntervalType => genHashCalendarInterval(input, result) case CalendarIntervalType => genHashCalendarInterval(input, result)
case _: DayTimeIntervalType => genHashLong(input, result) case _: DayTimeIntervalType => genHashLong(input, result)
case YearMonthIntervalType => genHashInt(input, result) case _: YearMonthIntervalType => genHashInt(input, result)
case BinaryType => genHashBytes(input, result) case BinaryType => genHashBytes(input, result)
case StringType => genHashString(input, result) case StringType => genHashString(input, result)
case ArrayType(et, containsNull) => genHashForArray(ctx, input, result, et, containsNull) case ArrayType(et, containsNull) => genHashForArray(ctx, input, result, et, containsNull)

View file

@ -125,11 +125,11 @@ object ExtractIntervalPart {
source: Expression, source: Expression,
errorHandleFunc: => Nothing): Expression = { errorHandleFunc: => Nothing): Expression = {
(extractField.toUpperCase(Locale.ROOT), source.dataType) match { (extractField.toUpperCase(Locale.ROOT), source.dataType) match {
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", YearMonthIntervalType) => case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", _: YearMonthIntervalType) =>
ExtractANSIIntervalYears(source) ExtractANSIIntervalYears(source)
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", CalendarIntervalType) => case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", CalendarIntervalType) =>
ExtractIntervalYears(source) ExtractIntervalYears(source)
case ("MONTH" | "MON" | "MONS" | "MONTHS", YearMonthIntervalType) => case ("MONTH" | "MON" | "MONS" | "MONTHS", _: YearMonthIntervalType) =>
ExtractANSIIntervalMonths(source) ExtractANSIIntervalMonths(source)
case ("MONTH" | "MON" | "MONS" | "MONTHS", CalendarIntervalType) => case ("MONTH" | "MON" | "MONS" | "MONTHS", CalendarIntervalType) =>
ExtractIntervalMonths(source) ExtractIntervalMonths(source)
@ -374,7 +374,7 @@ case class MakeYMInterval(years: Expression, months: Expression)
override def left: Expression = years override def left: Expression = years
override def right: Expression = months override def right: Expression = months
override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType) override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType)
override def dataType: DataType = YearMonthIntervalType override def dataType: DataType = YearMonthIntervalType()
override def nullSafeEval(year: Any, month: Any): Any = { override def nullSafeEval(year: Any, month: Any): Any = {
Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(), Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(),
@ -407,7 +407,7 @@ case class MultiplyYMInterval(
override def right: Expression = num override def right: Expression = num
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType) override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
override def dataType: DataType = YearMonthIntervalType override def dataType: DataType = YearMonthIntervalType()
@transient @transient
private lazy val evalFunc: (Int, Any) => Any = right.dataType match { private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
@ -517,7 +517,7 @@ case class DivideYMInterval(
override def right: Expression = num override def right: Expression = num
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType) override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
override def dataType: DataType = YearMonthIntervalType override def dataType: DataType = YearMonthIntervalType()
@transient @transient
private lazy val evalFunc: (Int, Any) => Any = right.dataType match { private lazy val evalFunc: (Int, Any) => Any = right.dataType match {

View file

@ -84,7 +84,7 @@ object Literal {
case ld: LocalDate => Literal(ld.toEpochDay.toInt, DateType) case ld: LocalDate => Literal(ld.toEpochDay.toInt, DateType)
case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType) case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
case d: Duration => Literal(durationToMicros(d), DayTimeIntervalType()) case d: Duration => Literal(durationToMicros(d), DayTimeIntervalType())
case p: Period => Literal(periodToMonths(p), YearMonthIntervalType) case p: Period => Literal(periodToMonths(p), YearMonthIntervalType())
case a: Array[Byte] => Literal(a, BinaryType) case a: Array[Byte] => Literal(a, BinaryType)
case a: collection.mutable.WrappedArray[_] => apply(a.array) case a: collection.mutable.WrappedArray[_] => apply(a.array)
case a: Array[_] => case a: Array[_] =>
@ -122,7 +122,7 @@ object Literal {
case _ if clz == classOf[Timestamp] => TimestampType case _ if clz == classOf[Timestamp] => TimestampType
case _ if clz == classOf[LocalDateTime] => TimestampWithoutTZType case _ if clz == classOf[LocalDateTime] => TimestampWithoutTZType
case _ if clz == classOf[Duration] => DayTimeIntervalType() case _ if clz == classOf[Duration] => DayTimeIntervalType()
case _ if clz == classOf[Period] => YearMonthIntervalType case _ if clz == classOf[Period] => YearMonthIntervalType()
case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT
case _ if clz == classOf[Array[Byte]] => BinaryType case _ if clz == classOf[Array[Byte]] => BinaryType
case _ if clz == classOf[Array[Char]] => StringType case _ if clz == classOf[Array[Char]] => StringType
@ -181,7 +181,7 @@ object Literal {
case TimestampType => create(0L, TimestampType) case TimestampType => create(0L, TimestampType)
case TimestampWithoutTZType => create(0L, TimestampWithoutTZType) case TimestampWithoutTZType => create(0L, TimestampWithoutTZType)
case it: DayTimeIntervalType => create(0L, it) case it: DayTimeIntervalType => create(0L, it)
case YearMonthIntervalType => create(0, YearMonthIntervalType) case it: YearMonthIntervalType => create(0, it)
case StringType => Literal("") case StringType => Literal("")
case BinaryType => Literal("".getBytes(StandardCharsets.UTF_8)) case BinaryType => Literal("".getBytes(StandardCharsets.UTF_8))
case CalendarIntervalType => Literal(new CalendarInterval(0, 0, 0)) case CalendarIntervalType => Literal(new CalendarInterval(0, 0, 0))
@ -200,7 +200,7 @@ object Literal {
case BooleanType => v.isInstanceOf[Boolean] case BooleanType => v.isInstanceOf[Boolean]
case ByteType => v.isInstanceOf[Byte] case ByteType => v.isInstanceOf[Byte]
case ShortType => v.isInstanceOf[Short] case ShortType => v.isInstanceOf[Short]
case IntegerType | DateType | YearMonthIntervalType => v.isInstanceOf[Int] case IntegerType | DateType | _: YearMonthIntervalType => v.isInstanceOf[Int]
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
v.isInstanceOf[Long] v.isInstanceOf[Long]
case FloatType => v.isInstanceOf[Float] case FloatType => v.isInstanceOf[Float]
@ -348,7 +348,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
TimestampFormatter.getFractionFormatter(timeZoneId).format(value.asInstanceOf[Long]) TimestampFormatter.getFractionFormatter(timeZoneId).format(value.asInstanceOf[Long])
case DayTimeIntervalType(startField, endField) => case DayTimeIntervalType(startField, endField) =>
toDayTimeIntervalString(value.asInstanceOf[Long], ANSI_STYLE, startField, endField) toDayTimeIntervalString(value.asInstanceOf[Long], ANSI_STYLE, startField, endField)
case YearMonthIntervalType => toYearMonthIntervalString(value.asInstanceOf[Int], ANSI_STYLE) case YearMonthIntervalType(startField, endField) =>
toYearMonthIntervalString(value.asInstanceOf[Int], ANSI_STYLE, startField, endField)
case _ => case _ =>
other.toString other.toString
} }
@ -401,7 +402,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
ExprCode.forNonNullValue(JavaCode.literal(code, dataType)) ExprCode.forNonNullValue(JavaCode.literal(code, dataType))
} }
dataType match { dataType match {
case BooleanType | IntegerType | DateType | YearMonthIntervalType => case BooleanType | IntegerType | DateType | _: YearMonthIntervalType =>
toExprCode(value.toString) toExprCode(value.toString)
case FloatType => case FloatType =>
value.asInstanceOf[Float] match { value.asInstanceOf[Float] match {
@ -471,7 +472,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'" case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'"
case (i: Long, DayTimeIntervalType(startField, endField)) => case (i: Long, DayTimeIntervalType(startField, endField)) =>
toDayTimeIntervalString(i, ANSI_STYLE, startField, endField) toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)
case (i: Int, YearMonthIntervalType) => toYearMonthIntervalString(i, ANSI_STYLE) case (i: Int, YearMonthIntervalType(startField, endField)) =>
toYearMonthIntervalString(i, ANSI_STYLE, startField, endField)
case _ => value.toString case _ => value.toString
} }
} }

View file

@ -101,9 +101,9 @@ case class WindowSpecDefinition(
private def isValidFrameType(ft: DataType): Boolean = (orderSpec.head.dataType, ft) match { private def isValidFrameType(ft: DataType): Boolean = (orderSpec.head.dataType, ft) match {
case (DateType, IntegerType) => true case (DateType, IntegerType) => true
case (DateType, YearMonthIntervalType) => true case (DateType, _: YearMonthIntervalType) => true
case (TimestampType, CalendarIntervalType) => true case (TimestampType, CalendarIntervalType) => true
case (TimestampType, YearMonthIntervalType) => true case (TimestampType, _: YearMonthIntervalType) => true
case (TimestampType, _: DayTimeIntervalType) => true case (TimestampType, _: DayTimeIntervalType) => true
case (a, b) => a == b case (a, b) => a == b
} }

View file

@ -2354,7 +2354,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT) val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT)
if (toUnit == "month") { if (toUnit == "month") {
assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0) assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0)
Literal(calendarInterval.months, YearMonthIntervalType) // TODO(SPARK-35773): Parse year-month interval literals to tightest types
Literal(calendarInterval.months, YearMonthIntervalType())
} else { } else {
assert(calendarInterval.months == 0) assert(calendarInterval.months == 0)
val fromUnit = val fromUnit =
@ -2513,7 +2514,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
} }
override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = { override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = {
YearMonthIntervalType // TODO(SPARK-35774): Parse any year-month interval types in SQL
YearMonthIntervalType()
} }
override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = { override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = {

View file

@ -106,7 +106,11 @@ object IntervalUtils {
private val yearMonthLiteralRegex = private val yearMonthLiteralRegex =
(s"(?i)^INTERVAL\\s+([+|-])?'$yearMonthPatternString'\\s+YEAR\\s+TO\\s+MONTH$$").r (s"(?i)^INTERVAL\\s+([+|-])?'$yearMonthPatternString'\\s+YEAR\\s+TO\\s+MONTH$$").r
def castStringToYMInterval(input: UTF8String): Int = { def castStringToYMInterval(
input: UTF8String,
// TODO(SPARK-35768): Take into account year-month interval fields in cast
startField: Byte,
endField: Byte): Int = {
input.trimAll().toString match { input.trimAll().toString match {
case yearMonthRegex("-", year, month) => toYMInterval(year, month, -1) case yearMonthRegex("-", year, month) => toYMInterval(year, month, -1)
case yearMonthRegex(_, year, month) => toYMInterval(year, month, 1) case yearMonthRegex(_, year, month) => toYMInterval(year, month, 1)
@ -934,9 +938,16 @@ object IntervalUtils {
* *
* @param months The number of months, positive or negative * @param months The number of months, positive or negative
* @param style The style of textual representation of the interval * @param style The style of textual representation of the interval
* @param startField The start field (YEAR or MONTH) which the interval comprises of.
* @param endField The end field (YEAR or MONTH) which the interval comprises of.
* @return Year-month interval string * @return Year-month interval string
*/ */
def toYearMonthIntervalString(months: Int, style: IntervalStyle): String = { def toYearMonthIntervalString(
months: Int,
style: IntervalStyle,
// TODO(SPARK-35771): Format year-month intervals using type fields
startField: Byte,
endField: Byte): String = {
var sign = "" var sign = ""
var absMonths: Long = months var absMonths: Long = months
if (months < 0) { if (months < 0) {
@ -956,6 +967,8 @@ object IntervalUtils {
* *
* @param micros The number of microseconds, positive or negative * @param micros The number of microseconds, positive or negative
* @param style The style of textual representation of the interval * @param style The style of textual representation of the interval
* @param startField The start field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
* @param endField The end field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
* @return Day-time interval string * @return Day-time interval string
*/ */
def toDayTimeIntervalString( def toDayTimeIntervalString(

View file

@ -63,7 +63,7 @@ object TypeUtils {
def checkForAnsiIntervalOrNumericType( def checkForAnsiIntervalOrNumericType(
dt: DataType, funcName: String): TypeCheckResult = dt match { dt: DataType, funcName: String): TypeCheckResult = dt match {
case YearMonthIntervalType | _: DayTimeIntervalType | NullType => case _: YearMonthIntervalType | _: DayTimeIntervalType | NullType =>
TypeCheckResult.TypeCheckSuccess TypeCheckResult.TypeCheckSuccess
case dt if dt.isInstanceOf[NumericType] => TypeCheckResult.TypeCheckSuccess case dt if dt.isInstanceOf[NumericType] => TypeCheckResult.TypeCheckSuccess
case other => TypeCheckResult.TypeCheckFailure( case other => TypeCheckResult.TypeCheckFailure(
@ -117,7 +117,7 @@ object TypeUtils {
def invokeOnceForInterval(dataType: DataType)(f: => Unit): Unit = { def invokeOnceForInterval(dataType: DataType)(f: => Unit): Unit = {
def isInterval(dataType: DataType): Boolean = dataType match { def isInterval(dataType: DataType): Boolean = dataType match {
case CalendarIntervalType | _: DayTimeIntervalType | YearMonthIntervalType => true case CalendarIntervalType | _: DayTimeIntervalType | _: YearMonthIntervalType => true
case _ => false case _ => false
} }
if (dataType.existsRecursively(isInterval)) f if (dataType.existsRecursively(isInterval)) f

View file

@ -1636,4 +1636,15 @@ private[spark] object QueryCompilationErrors {
def invalidDayTimeIntervalType(startFieldName: String, endFieldName: String): Throwable = { def invalidDayTimeIntervalType(startFieldName: String, endFieldName: String): Throwable = {
new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.") new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.")
} }
def invalidYearMonthField(field: Byte): Throwable = {
val supportedIds = YearMonthIntervalType.yearMonthFields
.map(i => s"$i (${YearMonthIntervalType.fieldToString(i)})")
new AnalysisException(s"Invalid field id '$field' in year-month interval. " +
s"Supported interval fields: ${supportedIds.mkString(", ")}.")
}
def invalidYearMonthIntervalType(startFieldName: String, endFieldName: String): Throwable = {
new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.")
}
} }

View file

@ -172,7 +172,9 @@ object DataType {
Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType, Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType, DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
// TODO(SPARK-35732): Parse DayTimeIntervalType from JSON // TODO(SPARK-35732): Parse DayTimeIntervalType from JSON
DayTimeIntervalType(), YearMonthIntervalType, TimestampWithoutTZType) DayTimeIntervalType(),
// TODO(SPARK-35770): Parse YearMonthIntervalType from JSON
YearMonthIntervalType(), TimestampWithoutTZType)
.map(t => t.typeName -> t).toMap .map(t => t.typeName -> t).toMap
} }

View file

@ -21,6 +21,8 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.Unstable import org.apache.spark.annotation.Unstable
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types.YearMonthIntervalType.fieldToString
/** /**
* The type represents year-month intervals of the SQL standard. A year-month interval is made up * The type represents year-month intervals of the SQL standard. A year-month interval is made up
@ -30,12 +32,15 @@ import org.apache.spark.annotation.Unstable
* *
* `YearMonthIntervalType` represents positive as well as negative year-month intervals. * `YearMonthIntervalType` represents positive as well as negative year-month intervals.
* *
* Please use the singleton `DataTypes.YearMonthIntervalType` to refer the type. * @param startField The leftmost field which the type comprises of. Valid values:
* 0 (YEAR), 1 (MONTH).
* @param endField The rightmost field which the type comprises of. Valid values:
* 0 (YEAR), 1 (MONTH).
* *
* @since 3.2.0 * @since 3.2.0
*/ */
@Unstable @Unstable
class YearMonthIntervalType private() extends AtomicType { case class YearMonthIntervalType(startField: Byte, endField: Byte) extends AtomicType {
/** /**
* Internally, values of year-month intervals are stored in `Int` values as amount of months * Internally, values of year-month intervals are stored in `Int` values as amount of months
* that are calculated by the formula: * that are calculated by the formula:
@ -55,16 +60,47 @@ class YearMonthIntervalType private() extends AtomicType {
private[spark] override def asNullable: YearMonthIntervalType = this private[spark] override def asNullable: YearMonthIntervalType = this
override def typeName: String = "interval year to month" override val typeName: String = {
val startFieldName = fieldToString(startField)
val endFieldName = fieldToString(endField)
if (startFieldName == endFieldName) {
s"interval $startFieldName"
} else if (startField < endField) {
s"interval $startFieldName to $endFieldName"
} else {
throw QueryCompilationErrors.invalidDayTimeIntervalType(startFieldName, endFieldName)
}
}
} }
/** /**
* The companion case object and its class is separated so the companion object also subclasses * Extra factory methods and pattern matchers for YearMonthIntervalType.
* the YearMonthIntervalType class. Otherwise, the companion object would be of type
* "YearMonthIntervalType$" in byte code. Defined with a private constructor so the companion object
* is the only possible instantiation.
* *
* @since 3.2.0 * @since 3.2.0
*/ */
@Unstable @Unstable
case object YearMonthIntervalType extends YearMonthIntervalType case object YearMonthIntervalType extends AbstractDataType {
val YEAR: Byte = 0
val MONTH: Byte = 1
val yearMonthFields = Seq(YEAR, MONTH)
def fieldToString(field: Byte): String = field match {
case YEAR => "year"
case MONTH => "month"
case invalid => throw QueryCompilationErrors.invalidYearMonthField(invalid)
}
val stringToField: Map[String, Byte] = yearMonthFields.map(i => fieldToString(i) -> i).toMap
val DEFAULT = YearMonthIntervalType(YEAR, MONTH)
def apply(): YearMonthIntervalType = DEFAULT
override private[sql] def defaultConcreteType: DataType = DEFAULT
override private[sql] def acceptsType(other: DataType): Boolean = {
other.isInstanceOf[YearMonthIntervalType]
}
override private[sql] def simpleString: String = defaultConcreteType.simpleString
}

View file

@ -54,7 +54,7 @@ private[sql] object ArrowUtils {
new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId) new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
} }
case NullType => ArrowType.Null.INSTANCE case NullType => ArrowType.Null.INSTANCE
case YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH) case _: YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
case _: DayTimeIntervalType => new ArrowType.Interval(IntervalUnit.DAY_TIME) case _: DayTimeIntervalType => new ArrowType.Interval(IntervalUnit.DAY_TIME)
case _ => case _ =>
throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}") throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
@ -76,7 +76,7 @@ private[sql] object ArrowUtils {
case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType
case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType
case ArrowType.Null.INSTANCE => NullType case ArrowType.Null.INSTANCE => NullType
case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType()
case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType() case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType()
case _ => throw new UnsupportedOperationException(s"Unsupported data type: $dt") case _ => throw new UnsupportedOperationException(s"Unsupported data type: $dt")
} }

View file

@ -284,7 +284,7 @@ object RandomDataGenerator {
new CalendarInterval(months, days, ns) new CalendarInterval(months, days, ns)
}) })
case _: DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS)) case _: DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS))
case YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized()) case _: YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized())
case DecimalType.Fixed(precision, scale) => Some( case DecimalType.Fixed(precision, scale) => Some(
() => BigDecimal.apply( () => BigDecimal.apply(
rand.nextLong() % math.pow(10, precision).toLong, rand.nextLong() % math.pow(10, precision).toLong,

View file

@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.CatalystTypeConverters
import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
/** /**
* Tests of [[RandomDataGenerator]]. * Tests of [[RandomDataGenerator]].
@ -146,7 +146,7 @@ class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
} }
test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") { test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") {
(dayTimeIntervalTypes :+ YearMonthIntervalType).foreach { dt => (dayTimeIntervalTypes ++ yearMonthIntervalTypes).foreach { dt =>
for (seed <- 1 to 1000) { for (seed <- 1 to 1000) {
val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get
val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt) val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt)

View file

@ -320,7 +320,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
val months = sign * input val months = sign * input
val period = IntervalUtils.monthsToPeriod(months) val period = IntervalUtils.monthsToPeriod(months)
assert( assert(
CatalystTypeConverters.createToScalaConverter(YearMonthIntervalType)(months) === period) CatalystTypeConverters.createToScalaConverter(YearMonthIntervalType())(months) === period)
} }
} }
} }

View file

@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils} import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils}
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
@SQLUserDefinedType(udt = classOf[ExamplePointUDT]) @SQLUserDefinedType(udt = classOf[ExamplePointUDT])
class ExamplePoint(val x: Double, val y: Double) extends Serializable { class ExamplePoint(val x: Double, val y: Double) extends Serializable {
@ -366,7 +366,8 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
} }
test("SPARK-34615: encoding/decoding YearMonthIntervalType to/from java.time.Period") { test("SPARK-34615: encoding/decoding YearMonthIntervalType to/from java.time.Period") {
val schema = new StructType().add("p", YearMonthIntervalType) yearMonthIntervalTypes.foreach { yearMonthIntervalType =>
val schema = new StructType().add("p", yearMonthIntervalType)
val encoder = RowEncoder(schema).resolveAndBind() val encoder = RowEncoder(schema).resolveAndBind()
val period = java.time.Period.ofMonths(1) val period = java.time.Period.ofMonths(1)
val row = toRow(encoder, Row(period)) val row = toRow(encoder, Row(period))
@ -374,6 +375,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
val readback = fromRow(encoder, row) val readback = fromRow(encoder, row)
assert(readback.get(0).equals(period)) assert(readback.get(0).equals(period))
} }
}
for { for {
elementType <- Seq(IntegerType, StringType) elementType <- Seq(IntegerType, StringType)

View file

@ -614,20 +614,20 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
Seq(true, false).foreach { failOnError => Seq(true, false).foreach { failOnError =>
checkExceptionInExpression[ArithmeticException]( checkExceptionInExpression[ArithmeticException](
UnaryMinus( UnaryMinus(
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType), Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType()),
failOnError), failOnError),
"overflow") "overflow")
checkExceptionInExpression[ArithmeticException]( checkExceptionInExpression[ArithmeticException](
Subtract( Subtract(
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType), Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType()),
Literal.create(Period.ofMonths(10), YearMonthIntervalType), Literal.create(Period.ofMonths(10), YearMonthIntervalType()),
failOnError failOnError
), ),
"overflow") "overflow")
checkExceptionInExpression[ArithmeticException]( checkExceptionInExpression[ArithmeticException](
Add( Add(
Literal.create(Period.ofMonths(Int.MaxValue), YearMonthIntervalType), Literal.create(Period.ofMonths(Int.MaxValue), YearMonthIntervalType()),
Literal.create(Period.ofMonths(10), YearMonthIntervalType), Literal.create(Period.ofMonths(10), YearMonthIntervalType()),
failOnError failOnError
), ),
"overflow") "overflow")

View file

@ -612,43 +612,43 @@ class CastSuite extends CastSuiteBase {
test("SPARK-35111: Cast string to year-month interval") { test("SPARK-35111: Cast string to year-month interval") {
checkEvaluation(cast(Literal.create("INTERVAL '1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL '1-0' YEAR TO MONTH"),
YearMonthIntervalType), 12) YearMonthIntervalType()), 12)
checkEvaluation(cast(Literal.create("INTERVAL '-1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL '-1-0' YEAR TO MONTH"),
YearMonthIntervalType), -12) YearMonthIntervalType()), -12)
checkEvaluation(cast(Literal.create("INTERVAL -'-1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL -'-1-0' YEAR TO MONTH"),
YearMonthIntervalType), 12) YearMonthIntervalType()), 12)
checkEvaluation(cast(Literal.create("INTERVAL +'-1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL +'-1-0' YEAR TO MONTH"),
YearMonthIntervalType), -12) YearMonthIntervalType()), -12)
checkEvaluation(cast(Literal.create("INTERVAL +'+1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL +'+1-0' YEAR TO MONTH"),
YearMonthIntervalType), 12) YearMonthIntervalType()), 12)
checkEvaluation(cast(Literal.create("INTERVAL +'1-0' YEAR TO MONTH"), checkEvaluation(cast(Literal.create("INTERVAL +'1-0' YEAR TO MONTH"),
YearMonthIntervalType), 12) YearMonthIntervalType()), 12)
checkEvaluation(cast(Literal.create(" interval +'1-0' YEAR TO MONTH "), checkEvaluation(cast(Literal.create(" interval +'1-0' YEAR TO MONTH "),
YearMonthIntervalType), 12) YearMonthIntervalType()), 12)
checkEvaluation(cast(Literal.create(" -1-0 "), YearMonthIntervalType), -12) checkEvaluation(cast(Literal.create(" -1-0 "), YearMonthIntervalType()), -12)
checkEvaluation(cast(Literal.create("-1-0"), YearMonthIntervalType), -12) checkEvaluation(cast(Literal.create("-1-0"), YearMonthIntervalType()), -12)
checkEvaluation(cast(Literal.create(null, StringType), YearMonthIntervalType), null) checkEvaluation(cast(Literal.create(null, StringType), YearMonthIntervalType()), null)
Seq("0-0", "10-1", "-178956970-7", "178956970-7", "-178956970-8").foreach { interval => Seq("0-0", "10-1", "-178956970-7", "178956970-7", "-178956970-8").foreach { interval =>
val ansiInterval = s"INTERVAL '$interval' YEAR TO MONTH" val ansiInterval = s"INTERVAL '$interval' YEAR TO MONTH"
checkEvaluation( checkEvaluation(
cast(cast(Literal.create(interval), YearMonthIntervalType), StringType), ansiInterval) cast(cast(Literal.create(interval), YearMonthIntervalType()), StringType), ansiInterval)
checkEvaluation(cast(cast(Literal.create(ansiInterval), checkEvaluation(cast(cast(Literal.create(ansiInterval),
YearMonthIntervalType), StringType), ansiInterval) YearMonthIntervalType()), StringType), ansiInterval)
} }
Seq("INTERVAL '-178956970-9' YEAR TO MONTH", "INTERVAL '178956970-8' YEAR TO MONTH") Seq("INTERVAL '-178956970-9' YEAR TO MONTH", "INTERVAL '178956970-8' YEAR TO MONTH")
.foreach { interval => .foreach { interval =>
val e = intercept[IllegalArgumentException] { val e = intercept[IllegalArgumentException] {
cast(Literal.create(interval), YearMonthIntervalType).eval() cast(Literal.create(interval), YearMonthIntervalType()).eval()
}.getMessage }.getMessage
assert(e.contains("Error parsing interval year-month string: integer overflow")) assert(e.contains("Error parsing interval year-month string: integer overflow"))
} }
Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue) Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue)
.foreach { period => .foreach { period =>
val interval = Literal.create(Period.ofMonths(period), YearMonthIntervalType) val interval = Literal.create(Period.ofMonths(period), YearMonthIntervalType())
checkEvaluation(cast(cast(interval, StringType), YearMonthIntervalType), period) checkEvaluation(cast(cast(interval, StringType), YearMonthIntervalType()), period)
} }
} }
} }

View file

@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils._
import org.apache.spark.sql.catalyst.util.IntervalUtils.microsToDuration import org.apache.spark.sql.catalyst.util.IntervalUtils.microsToDuration
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.unsafe.types.UTF8String
abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper { abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
@ -831,8 +831,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
s"INTERVAL '$intervalPayload' YEAR TO MONTH") s"INTERVAL '$intervalPayload' YEAR TO MONTH")
} }
yearMonthIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegen( checkConsistencyBetweenInterpretedAndCodegen(
(child: Expression) => Cast(child, StringType), YearMonthIntervalType) (child: Expression) => Cast(child, StringType), it)
}
} }
test("SPARK-34668: cast day-time interval to string") { test("SPARK-34668: cast day-time interval to string") {

View file

@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC} import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@ -526,7 +526,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
private def testAddMonths(dataType: DataType): Unit = { private def testAddMonths(dataType: DataType): Unit = {
def addMonths(date: Literal, months: Any): AddMonthsBase = dataType match { def addMonths(date: Literal, months: Any): AddMonthsBase = dataType match {
case IntegerType => AddMonths(date, Literal.create(months, dataType)) case IntegerType => AddMonths(date, Literal.create(months, dataType))
case YearMonthIntervalType => case _: YearMonthIntervalType =>
val period = if (months == null) null else Period.ofMonths(months.asInstanceOf[Int]) val period = if (months == null) null else Period.ofMonths(months.asInstanceOf[Int])
DateAddYMInterval(date, Literal.create(period, dataType)) DateAddYMInterval(date, Literal.create(period, dataType))
} }
@ -561,7 +561,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
test("SPARK-34721: add a year-month interval to a date") { test("SPARK-34721: add a year-month interval to a date") {
testAddMonths(YearMonthIntervalType) testAddMonths(YearMonthIntervalType())
// Test evaluation results between Interpreted mode and Codegen mode // Test evaluation results between Interpreted mode and Codegen mode
forAll ( forAll (
LiteralGenerator.randomGen(DateType), LiteralGenerator.randomGen(DateType),
@ -1596,18 +1596,21 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation( checkEvaluation(
TimestampAddYMInterval( TimestampAddYMInterval(
Literal(new Timestamp(sdf.parse("2016-01-29 10:00:00.000").getTime)), Literal(new Timestamp(sdf.parse("2016-01-29 10:00:00.000").getTime)),
Literal.create(null, YearMonthIntervalType), Literal.create(null, YearMonthIntervalType()),
timeZoneId), timeZoneId),
null) null)
checkEvaluation( checkEvaluation(
TimestampAddYMInterval( TimestampAddYMInterval(
Literal.create(null, TimestampType), Literal.create(null, TimestampType),
Literal.create(null, YearMonthIntervalType), Literal.create(null, YearMonthIntervalType()),
timeZoneId), timeZoneId),
null) null)
yearMonthIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegen( checkConsistencyBetweenInterpretedAndCodegen(
(ts: Expression, interval: Expression) => TimestampAddYMInterval(ts, interval, timeZoneId), (ts: Expression, interval: Expression) =>
TimestampType, YearMonthIntervalType) TimestampAddYMInterval(ts, interval, timeZoneId),
TimestampType, it)
}
} }
} }

View file

@ -699,7 +699,7 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") { test("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") {
val dayTime = Literal.create(Duration.ofSeconds(1237123123), DayTimeIntervalType()) val dayTime = Literal.create(Duration.ofSeconds(1237123123), DayTimeIntervalType())
val yearMonth = Literal.create(Period.ofMonths(1234), YearMonthIntervalType) val yearMonth = Literal.create(Period.ofMonths(1234), YearMonthIntervalType())
checkEvaluation(Murmur3Hash(Seq(dayTime), 10), -428664612) checkEvaluation(Murmur3Hash(Seq(dayTime), 10), -428664612)
checkEvaluation(Murmur3Hash(Seq(yearMonth), 10), -686520021) checkEvaluation(Murmur3Hash(Seq(yearMonth), 10), -686520021)
checkEvaluation(XxHash64(Seq(dayTime), 10), 8228802290839366895L) checkEvaluation(XxHash64(Seq(dayTime), 10), 8228802290839366895L)

View file

@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval} import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType} import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType}
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes} import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes, yearMonthIntervalTypes}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@ -280,6 +280,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
} }
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
test("SPARK-34824: multiply year-month interval by numeric") { test("SPARK-34824: multiply year-month interval by numeric") {
Seq( Seq(
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -307,9 +308,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
numericTypes.foreach { numType => numericTypes.foreach { numType =>
yearMonthIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegenAllowingException( checkConsistencyBetweenInterpretedAndCodegenAllowingException(
(interval: Expression, num: Expression) => MultiplyYMInterval(interval, num), (interval: Expression, num: Expression) => MultiplyYMInterval(interval, num),
YearMonthIntervalType, numType) it, numType)
}
} }
} }
@ -349,6 +352,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
} }
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
test("SPARK-34868: divide year-month interval by numeric") { test("SPARK-34868: divide year-month interval by numeric") {
Seq( Seq(
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null, (Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
@ -376,9 +380,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
} }
numericTypes.foreach { numType => numericTypes.foreach { numType =>
yearMonthIntervalTypes.foreach { it =>
checkConsistencyBetweenInterpretedAndCodegenAllowingException( checkConsistencyBetweenInterpretedAndCodegenAllowingException(
(interval: Expression, num: Expression) => DivideYMInterval(interval, num), (interval: Expression, num: Expression) => DivideYMInterval(interval, num),
YearMonthIntervalType, numType) it, numType)
}
} }
} }
@ -429,8 +435,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(ExtractANSIIntervalMonths(Literal(p)), checkEvaluation(ExtractANSIIntervalMonths(Literal(p)),
IntervalUtils.getMonths(p.toTotalMonths.toInt)) IntervalUtils.getMonths(p.toTotalMonths.toInt))
} }
checkEvaluation(ExtractANSIIntervalYears(Literal(null, YearMonthIntervalType)), null) checkEvaluation(ExtractANSIIntervalYears(Literal(null, YearMonthIntervalType())), null)
checkEvaluation(ExtractANSIIntervalMonths(Literal(null, YearMonthIntervalType)), null) checkEvaluation(ExtractANSIIntervalMonths(Literal(null, YearMonthIntervalType())), null)
} }
test("ANSI: extract days, hours, minutes and seconds") { test("ANSI: extract days, hours, minutes and seconds") {

View file

@ -50,7 +50,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Literal.create(null, DateType), null) checkEvaluation(Literal.create(null, DateType), null)
checkEvaluation(Literal.create(null, TimestampType), null) checkEvaluation(Literal.create(null, TimestampType), null)
checkEvaluation(Literal.create(null, CalendarIntervalType), null) checkEvaluation(Literal.create(null, CalendarIntervalType), null)
checkEvaluation(Literal.create(null, YearMonthIntervalType), null) checkEvaluation(Literal.create(null, YearMonthIntervalType()), null)
checkEvaluation(Literal.create(null, DayTimeIntervalType()), null) checkEvaluation(Literal.create(null, DayTimeIntervalType()), null)
checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null) checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null)
checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null) checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null)
@ -79,7 +79,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0)) checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
} }
checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L)) checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L))
checkEvaluation(Literal.default(YearMonthIntervalType), 0) checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
checkEvaluation(Literal.default(DayTimeIntervalType()), 0L) checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
checkEvaluation(Literal.default(ArrayType(StringType)), Array()) checkEvaluation(Literal.default(ArrayType(StringType)), Array())
checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map()) checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
@ -345,7 +345,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
assert(Literal(Array("1", "2", "3")) == assert(Literal(Array("1", "2", "3")) ==
Literal.create(Array("1", "2", "3"), ArrayType(StringType))) Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
assert(Literal(Array(Period.ofMonths(1))) == assert(Literal(Array(Period.ofMonths(1))) ==
Literal.create(Array(Period.ofMonths(1)), ArrayType(YearMonthIntervalType))) Literal.create(Array(Period.ofMonths(1)), ArrayType(YearMonthIntervalType())))
} }
test("SPARK-34342: Date/Timestamp toString") { test("SPARK-34342: Date/Timestamp toString") {

View file

@ -184,7 +184,7 @@ object LiteralGenerator {
lazy val yearMonthIntervalLiteralGen: Gen[Literal] = { lazy val yearMonthIntervalLiteralGen: Gen[Literal] = {
for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) } for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) }
yield Literal.create(Period.ofMonths(months), YearMonthIntervalType) yield Literal.create(Period.ofMonths(months), YearMonthIntervalType())
} }
def randomGen(dt: DataType): Gen[Literal] = { def randomGen(dt: DataType): Gen[Literal] = {
@ -204,7 +204,7 @@ object LiteralGenerator {
case CalendarIntervalType => calendarIntervalLiterGen case CalendarIntervalType => calendarIntervalLiterGen
case DecimalType.Fixed(precision, scale) => decimalLiteralGen(precision, scale) case DecimalType.Fixed(precision, scale) => decimalLiteralGen(precision, scale)
case _: DayTimeIntervalType => dayTimeIntervalLiteralGen case _: DayTimeIntervalType => dayTimeIntervalLiteralGen
case YearMonthIntervalType => yearMonthIntervalLiteralGen case _: YearMonthIntervalType => yearMonthIntervalLiteralGen
case dt => throw new IllegalArgumentException(s"not supported type $dt") case dt => throw new IllegalArgumentException(s"not supported type $dt")
} }
} }

View file

@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
import org.apache.spark.sql.catalyst.util.IntervalUtils import org.apache.spark.sql.catalyst.util.IntervalUtils
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.Platform
import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.unsafe.types.UTF8String
@ -31,7 +31,7 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
val fixedLengthTypes = Array[DataType]( val fixedLengthTypes = Array[DataType](
BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
DateType, TimestampType, YearMonthIntervalType) ++ dayTimeIntervalTypes DateType, TimestampType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
val variableLengthTypes = Array( val variableLengthTypes = Array(
StringType, DecimalType.defaultConcreteType, CalendarIntervalType, BinaryType, StringType, DecimalType.defaultConcreteType, CalendarIntervalType, BinaryType,
@ -44,16 +44,18 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
testBothCodegenAndInterpreted("fixed-length types") { testBothCodegenAndInterpreted("fixed-length types") {
val inputRow = InternalRow.fromSeq(Seq( val inputRow = InternalRow.fromSeq(Seq(
true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L, Int.MaxValue) ++ true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L) ++
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue)) Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue) ++
Seq.tabulate(yearMonthIntervalTypes.length)(_ => Int.MaxValue))
val proj = createMutableProjection(fixedLengthTypes) val proj = createMutableProjection(fixedLengthTypes)
assert(proj(inputRow) === inputRow) assert(proj(inputRow) === inputRow)
} }
testBothCodegenAndInterpreted("unsafe buffer") { testBothCodegenAndInterpreted("unsafe buffer") {
val inputRow = InternalRow.fromSeq(Seq( val inputRow = InternalRow.fromSeq(Seq(
false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L, Int.MinValue) ++ false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L) ++
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue)) Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue) ++
Seq.tabulate(yearMonthIntervalTypes.length)(_ => Int.MaxValue))
val numFields = fixedLengthTypes.length val numFields = fixedLengthTypes.length
val numBytes = Platform.BYTE_ARRAY_OFFSET + UnsafeRow.calculateBitSetWidthInBytes(numFields) + val numBytes = Platform.BYTE_ARRAY_OFFSET + UnsafeRow.calculateBitSetWidthInBytes(numFields) +
UnsafeRow.WORD_SIZE * numFields UnsafeRow.WORD_SIZE * numFields

View file

@ -275,8 +275,8 @@ class PushFoldableIntoBranchesSuite
Literal(new CalendarInterval(1, 2, 0)))), Literal(new CalendarInterval(1, 2, 0)))),
If(a, Literal(Date.valueOf("2021-02-02")), Literal(Date.valueOf("2021-02-03")))) If(a, Literal(Date.valueOf("2021-02-02")), Literal(Date.valueOf("2021-02-03"))))
assertEquivalent(DateAddYMInterval(Literal(d), assertEquivalent(DateAddYMInterval(Literal(d),
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType), If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType()),
Literal.create(Period.ofMonths(2), YearMonthIntervalType))), Literal.create(Period.ofMonths(2), YearMonthIntervalType()))),
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))) If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01"))))
assertEquivalent(DateDiff(Literal(d), assertEquivalent(DateDiff(Literal(d),
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))), If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))),
@ -286,8 +286,8 @@ class PushFoldableIntoBranchesSuite
If(a, Literal(Date.valueOf("2020-12-31")), Literal(Date.valueOf("2020-12-30")))) If(a, Literal(Date.valueOf("2020-12-31")), Literal(Date.valueOf("2020-12-30"))))
assertEquivalent(TimestampAddYMInterval( assertEquivalent(TimestampAddYMInterval(
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType), Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType), If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType()),
Literal.create(Period.ofMonths(2), YearMonthIntervalType))), Literal.create(Period.ofMonths(2), YearMonthIntervalType()))),
If(a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType), If(a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType),
Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))) Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType)))
assertEquivalent(TimeAdd( assertEquivalent(TimeAdd(
@ -312,8 +312,8 @@ class PushFoldableIntoBranchesSuite
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-02"))), CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-02"))),
(c, Literal(Date.valueOf("2021-02-03")))), None)) (c, Literal(Date.valueOf("2021-02-03")))), None))
assertEquivalent(DateAddYMInterval(Literal(d), assertEquivalent(DateAddYMInterval(Literal(d),
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)), CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType())),
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)), (c, Literal.create(Period.ofMonths(2), YearMonthIntervalType()))), None)),
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))), CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
(c, Literal(Date.valueOf("2021-03-01")))), None)) (c, Literal(Date.valueOf("2021-03-01")))), None))
assertEquivalent(DateDiff(Literal(d), assertEquivalent(DateDiff(Literal(d),
@ -326,8 +326,8 @@ class PushFoldableIntoBranchesSuite
(c, Literal(Date.valueOf("2020-12-30")))), None)) (c, Literal(Date.valueOf("2020-12-30")))), None))
assertEquivalent(TimestampAddYMInterval( assertEquivalent(TimestampAddYMInterval(
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType), Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)), CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType())),
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)), (c, Literal.create(Period.ofMonths(2), YearMonthIntervalType()))), None)),
CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType)), CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType)),
(c, Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))), None)) (c, Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))), None))
assertEquivalent(TimeAdd( assertEquivalent(TimeAdd(

View file

@ -63,7 +63,7 @@ class DataTypeParserSuite extends SparkFunSuite {
checkDataType("BINARY", BinaryType) checkDataType("BINARY", BinaryType)
checkDataType("void", NullType) checkDataType("void", NullType)
checkDataType("interval", CalendarIntervalType) checkDataType("interval", CalendarIntervalType)
checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType) checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType())
checkDataType("interval day to second", DayTimeIntervalType()) checkDataType("interval day to second", DayTimeIntervalType())
checkDataType("array<doublE>", ArrayType(DoubleType, true)) checkDataType("array<doublE>", ArrayType(DoubleType, true))

View file

@ -503,6 +503,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
} }
test("SPARK-35016: format year-month intervals") { test("SPARK-35016: format year-month intervals") {
import org.apache.spark.sql.types.YearMonthIntervalType._
Seq( Seq(
0 -> ("0-0", "INTERVAL '0-0' YEAR TO MONTH"), 0 -> ("0-0", "INTERVAL '0-0' YEAR TO MONTH"),
-11 -> ("-0-11", "INTERVAL '-0-11' YEAR TO MONTH"), -11 -> ("-0-11", "INTERVAL '-0-11' YEAR TO MONTH"),
@ -514,8 +515,8 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
Int.MinValue -> ("-178956970-8", "INTERVAL '-178956970-8' YEAR TO MONTH"), Int.MinValue -> ("-178956970-8", "INTERVAL '-178956970-8' YEAR TO MONTH"),
Int.MaxValue -> ("178956970-7", "INTERVAL '178956970-7' YEAR TO MONTH") Int.MaxValue -> ("178956970-7", "INTERVAL '178956970-7' YEAR TO MONTH")
).foreach { case (months, (hiveIntervalStr, ansiIntervalStr)) => ).foreach { case (months, (hiveIntervalStr, ansiIntervalStr)) =>
assert(toYearMonthIntervalString(months, ANSI_STYLE) === ansiIntervalStr) assert(toYearMonthIntervalString(months, ANSI_STYLE, YEAR, MONTH) === ansiIntervalStr)
assert(toYearMonthIntervalString(months, HIVE_STYLE) === hiveIntervalStr) assert(toYearMonthIntervalString(months, HIVE_STYLE, YEAR, MONTH) === hiveIntervalStr)
} }
} }

View file

@ -22,7 +22,7 @@ import com.fasterxml.jackson.core.JsonParseException
import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
class DataTypeSuite extends SparkFunSuite { class DataTypeSuite extends SparkFunSuite {
@ -256,7 +256,7 @@ class DataTypeSuite extends SparkFunSuite {
checkDataTypeFromJson(VarcharType(10)) checkDataTypeFromJson(VarcharType(10))
checkDataTypeFromDDL(VarcharType(11)) checkDataTypeFromDDL(VarcharType(11))
checkDataTypeFromDDL(YearMonthIntervalType) checkDataTypeFromDDL(YearMonthIntervalType())
dayTimeIntervalTypes.foreach(checkDataTypeFromDDL) dayTimeIntervalTypes.foreach(checkDataTypeFromDDL)
val metadata = new MetadataBuilder() val metadata = new MetadataBuilder()
@ -325,7 +325,7 @@ class DataTypeSuite extends SparkFunSuite {
checkDefaultSize(CharType(100), 100) checkDefaultSize(CharType(100), 100)
checkDefaultSize(VarcharType(5), 5) checkDefaultSize(VarcharType(5), 5)
checkDefaultSize(VarcharType(10), 10) checkDefaultSize(VarcharType(10), 10)
checkDefaultSize(YearMonthIntervalType, 4) yearMonthIntervalTypes.foreach(checkDefaultSize(_, 4))
dayTimeIntervalTypes.foreach(checkDefaultSize(_, 8)) dayTimeIntervalTypes.foreach(checkDefaultSize(_, 8))
def checkEqualsIgnoreCompatibleNullability( def checkEqualsIgnoreCompatibleNullability(

View file

@ -18,6 +18,7 @@
package org.apache.spark.sql.types package org.apache.spark.sql.types
import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, MINUTE, SECOND} import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, MINUTE, SECOND}
import org.apache.spark.sql.types.YearMonthIntervalType.{MONTH, YEAR}
/** /**
* Utility functions for working with DataTypes in tests. * Utility functions for working with DataTypes in tests.
@ -63,12 +64,16 @@ object DataTypeTestUtils {
DayTimeIntervalType(MINUTE, SECOND), DayTimeIntervalType(MINUTE, SECOND),
DayTimeIntervalType(SECOND, SECOND)) DayTimeIntervalType(SECOND, SECOND))
val yearMonthIntervalTypes: Seq[YearMonthIntervalType] = Seq(
YearMonthIntervalType(YEAR, MONTH),
YearMonthIntervalType(YEAR, YEAR),
YearMonthIntervalType(MONTH, MONTH))
/** /**
* Instances of all [[NumericType]]s and [[CalendarIntervalType]] * Instances of all [[NumericType]]s and [[CalendarIntervalType]]
*/ */
val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal ++ Set( val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal ++
CalendarIntervalType, Set(CalendarIntervalType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
YearMonthIntervalType) ++ dayTimeIntervalTypes
/** /**
* All the types that support ordering * All the types that support ordering
@ -79,8 +84,7 @@ object DataTypeTestUtils {
TimestampWithoutTZType, TimestampWithoutTZType,
DateType, DateType,
StringType, StringType,
BinaryType, BinaryType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
YearMonthIntervalType) ++ dayTimeIntervalTypes
/** /**
* All the types that we can use in a property check * All the types that we can use in a property check
@ -96,9 +100,7 @@ object DataTypeTestUtils {
DateType, DateType,
StringType, StringType,
TimestampType, TimestampType,
TimestampWithoutTZType, TimestampWithoutTZType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
YearMonthIntervalType
) ++ dayTimeIntervalTypes
/** /**
* Instances of [[ArrayType]] for all [[AtomicType]]s. Arrays of these types may contain null. * Instances of [[ArrayType]] for all [[AtomicType]]s. Arrays of these types may contain null.

View file

@ -48,7 +48,7 @@ class ArrowUtilsSuite extends SparkFunSuite {
roundtrip(BinaryType) roundtrip(BinaryType)
roundtrip(DecimalType.SYSTEM_DEFAULT) roundtrip(DecimalType.SYSTEM_DEFAULT)
roundtrip(DateType) roundtrip(DateType)
roundtrip(YearMonthIntervalType) roundtrip(YearMonthIntervalType())
roundtrip(DayTimeIntervalType()) roundtrip(DayTimeIntervalType())
val tsExMsg = intercept[UnsupportedOperationException] { val tsExMsg = intercept[UnsupportedOperationException] {
roundtrip(TimestampType) roundtrip(TimestampType)

View file

@ -117,8 +117,8 @@ object HiveResult {
struct.toSeq.zip(fields).map { case (v, t) => struct.toSeq.zip(fields).map { case (v, t) =>
s""""${t.name}":${toHiveString((v, t.dataType), true, formatters)}""" s""""${t.name}":${toHiveString((v, t.dataType), true, formatters)}"""
}.mkString("{", ",", "}") }.mkString("{", ",", "}")
case (period: Period, YearMonthIntervalType) => case (period: Period, YearMonthIntervalType(startField, endField)) =>
toYearMonthIntervalString(periodToMonths(period), HIVE_STYLE) toYearMonthIntervalString(periodToMonths(period), HIVE_STYLE, startField, endField)
case (duration: Duration, DayTimeIntervalType(startField, endField)) => case (duration: Duration, DayTimeIntervalType(startField, endField)) =>
toDayTimeIntervalString(durationToMicros(duration), HIVE_STYLE, startField, endField) toDayTimeIntervalString(durationToMicros(duration), HIVE_STYLE, startField, endField)
case (other, _: UserDefinedType[_]) => other.toString case (other, _: UserDefinedType[_]) => other.toString

View file

@ -158,7 +158,8 @@ abstract class HashMapGenerator(
dataType match { dataType match {
case BooleanType => hashInt(s"$input ? 1 : 0") case BooleanType => hashInt(s"$input ? 1 : 0")
case ByteType | ShortType | IntegerType | DateType | YearMonthIntervalType => hashInt(input) case ByteType | ShortType | IntegerType | DateType | _: YearMonthIntervalType =>
hashInt(input)
case LongType | TimestampType | _: DayTimeIntervalType => hashLong(input) case LongType | TimestampType | _: DayTimeIntervalType => hashLong(input)
case FloatType => hashInt(s"Float.floatToIntBits($input)") case FloatType => hashInt(s"Float.floatToIntBits($input)")
case DoubleType => hashLong(s"Double.doubleToLongBits($input)") case DoubleType => hashLong(s"Double.doubleToLongBits($input)")

View file

@ -87,7 +87,7 @@ sealed trait BufferSetterGetterUtils {
(row: InternalRow, ordinal: Int) => (row: InternalRow, ordinal: Int) =>
if (row.isNullAt(ordinal)) null else row.getLong(ordinal) if (row.isNullAt(ordinal)) null else row.getLong(ordinal)
case YearMonthIntervalType => case _: YearMonthIntervalType =>
(row: InternalRow, ordinal: Int) => (row: InternalRow, ordinal: Int) =>
if (row.isNullAt(ordinal)) null else row.getInt(ordinal) if (row.isNullAt(ordinal)) null else row.getInt(ordinal)
@ -195,7 +195,7 @@ sealed trait BufferSetterGetterUtils {
row.setNullAt(ordinal) row.setNullAt(ordinal)
} }
case YearMonthIntervalType => case _: YearMonthIntervalType =>
(row: InternalRow, ordinal: Int, value: Any) => (row: InternalRow, ordinal: Int, value: Any) =>
if (value != null) { if (value != null) {
row.setInt(ordinal, value.asInstanceOf[Int]) row.setInt(ordinal, value.asInstanceOf[Int])

View file

@ -75,7 +75,7 @@ object ArrowWriter {
} }
new StructWriter(vector, children.toArray) new StructWriter(vector, children.toArray)
case (NullType, vector: NullVector) => new NullWriter(vector) case (NullType, vector: NullVector) => new NullWriter(vector)
case (YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector) case (_: YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector)
case (_: DayTimeIntervalType, vector: IntervalDayVector) => new IntervalDayWriter(vector) case (_: DayTimeIntervalType, vector: IntervalDayVector) => new IntervalDayWriter(vector)
case (dt, _) => case (dt, _) =>
throw QueryExecutionErrors.unsupportedDataTypeError(dt) throw QueryExecutionErrors.unsupportedDataTypeError(dt)

View file

@ -139,7 +139,7 @@ private[sql] object ColumnAccessor {
case BooleanType => new BooleanColumnAccessor(buf) case BooleanType => new BooleanColumnAccessor(buf)
case ByteType => new ByteColumnAccessor(buf) case ByteType => new ByteColumnAccessor(buf)
case ShortType => new ShortColumnAccessor(buf) case ShortType => new ShortColumnAccessor(buf)
case IntegerType | DateType | YearMonthIntervalType => new IntColumnAccessor(buf) case IntegerType | DateType | _: YearMonthIntervalType => new IntColumnAccessor(buf)
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnAccessor(buf) case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnAccessor(buf)
case FloatType => new FloatColumnAccessor(buf) case FloatType => new FloatColumnAccessor(buf)
case DoubleType => new DoubleColumnAccessor(buf) case DoubleType => new DoubleColumnAccessor(buf)

View file

@ -174,7 +174,7 @@ private[columnar] object ColumnBuilder {
case BooleanType => new BooleanColumnBuilder case BooleanType => new BooleanColumnBuilder
case ByteType => new ByteColumnBuilder case ByteType => new ByteColumnBuilder
case ShortType => new ShortColumnBuilder case ShortType => new ShortColumnBuilder
case IntegerType | DateType | YearMonthIntervalType => new IntColumnBuilder case IntegerType | DateType | _: YearMonthIntervalType => new IntColumnBuilder
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnBuilder case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnBuilder
case FloatType => new FloatColumnBuilder case FloatType => new FloatColumnBuilder
case DoubleType => new DoubleColumnBuilder case DoubleType => new DoubleColumnBuilder

View file

@ -257,7 +257,7 @@ private[columnar] object LONG extends NativeColumnType(LongType, 8) {
} }
} }
private[columnar] object YEAR_MONTH_INTERVAL extends NativeColumnType(YearMonthIntervalType, 4) { private[columnar] object YEAR_MONTH_INTERVAL extends NativeColumnType(YearMonthIntervalType(), 4) {
override def append(v: Int, buffer: ByteBuffer): Unit = { override def append(v: Int, buffer: ByteBuffer): Unit = {
buffer.putInt(v) buffer.putInt(v)
} }
@ -817,7 +817,7 @@ private[columnar] object ColumnType {
case BooleanType => BOOLEAN case BooleanType => BOOLEAN
case ByteType => BYTE case ByteType => BYTE
case ShortType => SHORT case ShortType => SHORT
case IntegerType | DateType | YearMonthIntervalType => INT case IntegerType | DateType | _: YearMonthIntervalType => INT
case LongType | TimestampType | _: DayTimeIntervalType => LONG case LongType | TimestampType | _: DayTimeIntervalType => LONG
case FloatType => FLOAT case FloatType => FLOAT
case DoubleType => DOUBLE case DoubleType => DOUBLE

View file

@ -80,7 +80,7 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
case BooleanType => classOf[BooleanColumnAccessor].getName case BooleanType => classOf[BooleanColumnAccessor].getName
case ByteType => classOf[ByteColumnAccessor].getName case ByteType => classOf[ByteColumnAccessor].getName
case ShortType => classOf[ShortColumnAccessor].getName case ShortType => classOf[ShortColumnAccessor].getName
case IntegerType | DateType | YearMonthIntervalType => classOf[IntColumnAccessor].getName case IntegerType | DateType | _: YearMonthIntervalType => classOf[IntColumnAccessor].getName
case LongType | TimestampType | _: DayTimeIntervalType => case LongType | TimestampType | _: DayTimeIntervalType =>
classOf[LongColumnAccessor].getName classOf[LongColumnAccessor].getName
case FloatType => classOf[FloatColumnAccessor].getName case FloatType => classOf[FloatColumnAccessor].getName

View file

@ -95,9 +95,9 @@ trait WindowExecBase extends UnaryExecNode {
// Create the projection which returns the current 'value' modified by adding the offset. // Create the projection which returns the current 'value' modified by adding the offset.
val boundExpr = (expr.dataType, boundOffset.dataType) match { val boundExpr = (expr.dataType, boundOffset.dataType) match {
case (DateType, IntegerType) => DateAdd(expr, boundOffset) case (DateType, IntegerType) => DateAdd(expr, boundOffset)
case (DateType, YearMonthIntervalType) => DateAddYMInterval(expr, boundOffset) case (DateType, _: YearMonthIntervalType) => DateAddYMInterval(expr, boundOffset)
case (TimestampType, CalendarIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone)) case (TimestampType, CalendarIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
case (TimestampType, YearMonthIntervalType) => case (TimestampType, _: YearMonthIntervalType) =>
TimestampAddYMInterval(expr, boundOffset, Some(timeZone)) TimestampAddYMInterval(expr, boundOffset, Some(timeZone))
case (TimestampType, _: DayTimeIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone)) case (TimestampType, _: DayTimeIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
case (a, b) if a == b => Add(expr, boundOffset) case (a, b) if a == b => Add(expr, boundOffset)

View file

@ -1129,7 +1129,9 @@ class DataFrameAggregateSuite extends QueryTest
val sumDF = df.select(sum($"year-month"), sum($"day-time")) val sumDF = df.select(sum($"year-month"), sum($"day-time"))
checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0))) checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined) assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(sumDF.schema == StructType(Seq(StructField("sum(year-month)", YearMonthIntervalType), assert(sumDF.schema == StructType(Seq(
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
StructField("sum(year-month)", YearMonthIntervalType()),
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions // TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
StructField("sum(day-time)", DayTimeIntervalType())))) StructField("sum(day-time)", DayTimeIntervalType()))))
@ -1139,7 +1141,8 @@ class DataFrameAggregateSuite extends QueryTest
Row(3, Period.of(1, 6, 0), Duration.ofDays(-11)) :: Nil) Row(3, Period.of(1, 6, 0), Duration.ofDays(-11)) :: Nil)
assert(find(sumDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined) assert(find(sumDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(sumDF2.schema == StructType(Seq(StructField("class", IntegerType, false), assert(sumDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
StructField("sum(year-month)", YearMonthIntervalType), // TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
StructField("sum(year-month)", YearMonthIntervalType()),
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions // TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
StructField("sum(day-time)", DayTimeIntervalType())))) StructField("sum(day-time)", DayTimeIntervalType()))))
@ -1169,7 +1172,9 @@ class DataFrameAggregateSuite extends QueryTest
val avgDF = df.select(avg($"year-month"), avg($"day-time")) val avgDF = df.select(avg($"year-month"), avg($"day-time"))
checkAnswer(avgDF, Row(Period.ofMonths(7), Duration.ofDays(0))) checkAnswer(avgDF, Row(Period.ofMonths(7), Duration.ofDays(0)))
assert(find(avgDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined) assert(find(avgDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(avgDF.schema == StructType(Seq(StructField("avg(year-month)", YearMonthIntervalType), assert(avgDF.schema == StructType(Seq(
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
StructField("avg(year-month)", YearMonthIntervalType()),
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions // TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
StructField("avg(day-time)", DayTimeIntervalType())))) StructField("avg(day-time)", DayTimeIntervalType()))))
@ -1179,7 +1184,8 @@ class DataFrameAggregateSuite extends QueryTest
Row(3, Period.ofMonths(9), Duration.ofDays(-5).plusHours(-12)) :: Nil) Row(3, Period.ofMonths(9), Duration.ofDays(-5).plusHours(-12)) :: Nil)
assert(find(avgDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined) assert(find(avgDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(avgDF2.schema == StructType(Seq(StructField("class", IntegerType, false), assert(avgDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
StructField("avg(year-month)", YearMonthIntervalType), // TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
StructField("avg(year-month)", YearMonthIntervalType()),
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions // TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
StructField("avg(day-time)", DayTimeIntervalType())))) StructField("avg(day-time)", DayTimeIntervalType()))))

View file

@ -908,18 +908,21 @@ class UDFSuite extends QueryTest with SharedSparkSession {
val incMonth = udf((p: java.time.Period) => p.plusMonths(1)) val incMonth = udf((p: java.time.Period) => p.plusMonths(1))
val result = input.select(incMonth($"p").as("new_p")) val result = input.select(incMonth($"p").as("new_p"))
checkAnswer(result, Row(java.time.Period.ofYears(1)) :: Nil) checkAnswer(result, Row(java.time.Period.ofYears(1)) :: Nil)
assert(result.schema === new StructType().add("new_p", YearMonthIntervalType)) // TODO(SPARK-35777): Check all year-month interval types in UDF
assert(result.schema === new StructType().add("new_p", YearMonthIntervalType()))
// UDF produces `null` // UDF produces `null`
val nullFunc = udf((_: java.time.Period) => null.asInstanceOf[java.time.Period]) val nullFunc = udf((_: java.time.Period) => null.asInstanceOf[java.time.Period])
val nullResult = input.select(nullFunc($"p").as("null_p")) val nullResult = input.select(nullFunc($"p").as("null_p"))
checkAnswer(nullResult, Row(null) :: Nil) checkAnswer(nullResult, Row(null) :: Nil)
assert(nullResult.schema === new StructType().add("null_p", YearMonthIntervalType)) // TODO(SPARK-35777): Check all year-month interval types in UDF
assert(nullResult.schema === new StructType().add("null_p", YearMonthIntervalType()))
// Input parameter of UDF is null // Input parameter of UDF is null
val nullInput = Seq(null.asInstanceOf[java.time.Period]).toDF("null_p") val nullInput = Seq(null.asInstanceOf[java.time.Period]).toDF("null_p")
val constPeriod = udf((_: java.time.Period) => java.time.Period.ofYears(10)) val constPeriod = udf((_: java.time.Period) => java.time.Period.ofYears(10))
val constResult = nullInput.select(constPeriod($"null_p").as("10_years")) val constResult = nullInput.select(constPeriod($"null_p").as("10_years"))
checkAnswer(constResult, Row(java.time.Period.ofYears(10)) :: Nil) checkAnswer(constResult, Row(java.time.Period.ofYears(10)) :: Nil)
assert(constResult.schema === new StructType().add("10_years", YearMonthIntervalType)) // TODO(SPARK-35777): Check all year-month interval types in UDF
assert(constResult.schema === new StructType().add("10_years", YearMonthIntervalType()))
// Error in the conversion of UDF result to the internal representation of year-month interval // Error in the conversion of UDF result to the internal representation of year-month interval
val overflowFunc = udf((p: java.time.Period) => p.plusYears(Long.MaxValue)) val overflowFunc = udf((p: java.time.Period) => p.plusYears(Long.MaxValue))
val e = intercept[SparkException] { val e = intercept[SparkException] {

View file

@ -56,7 +56,7 @@ class ArrowWriterSuite extends SparkFunSuite {
case BinaryType => reader.getBinary(rowId) case BinaryType => reader.getBinary(rowId)
case DateType => reader.getInt(rowId) case DateType => reader.getInt(rowId)
case TimestampType => reader.getLong(rowId) case TimestampType => reader.getLong(rowId)
case YearMonthIntervalType => reader.getInt(rowId) case _: YearMonthIntervalType => reader.getInt(rowId)
case _: DayTimeIntervalType => reader.getLong(rowId) case _: DayTimeIntervalType => reader.getLong(rowId)
} }
assert(value === datum) assert(value === datum)
@ -77,7 +77,7 @@ class ArrowWriterSuite extends SparkFunSuite {
check(DateType, Seq(0, 1, 2, null, 4)) check(DateType, Seq(0, 1, 2, null, 4))
check(TimestampType, Seq(0L, 3.6e9.toLong, null, 8.64e10.toLong), "America/Los_Angeles") check(TimestampType, Seq(0L, 3.6e9.toLong, null, 8.64e10.toLong), "America/Los_Angeles")
check(NullType, Seq(null, null, null)) check(NullType, Seq(null, null, null))
check(YearMonthIntervalType, Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue)) check(YearMonthIntervalType(), Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue))
check(DayTimeIntervalType(), Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L), check(DayTimeIntervalType(), Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L),
(Long.MinValue + 808L))) (Long.MinValue + 808L)))
} }
@ -128,7 +128,7 @@ class ArrowWriterSuite extends SparkFunSuite {
case DoubleType => reader.getDoubles(0, data.size) case DoubleType => reader.getDoubles(0, data.size)
case DateType => reader.getInts(0, data.size) case DateType => reader.getInts(0, data.size)
case TimestampType => reader.getLongs(0, data.size) case TimestampType => reader.getLongs(0, data.size)
case YearMonthIntervalType => reader.getInts(0, data.size) case _: YearMonthIntervalType => reader.getInts(0, data.size)
case _: DayTimeIntervalType => reader.getLongs(0, data.size) case _: DayTimeIntervalType => reader.getLongs(0, data.size)
} }
assert(values === data) assert(values === data)
@ -144,7 +144,8 @@ class ArrowWriterSuite extends SparkFunSuite {
check(DoubleType, (0 until 10).map(_.toDouble)) check(DoubleType, (0 until 10).map(_.toDouble))
check(DateType, (0 until 10)) check(DateType, (0 until 10))
check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles") check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles")
check(YearMonthIntervalType, (0 until 10)) // TODO(SPARK-35776): Check all year-month interval types in arrow
check(YearMonthIntervalType(), (0 until 10))
// TODO(SPARK-35731): Check all day-time interval types in arrow // TODO(SPARK-35731): Check all day-time interval types in arrow
check(DayTimeIntervalType(), (-10 until 10).map(_ * 1000.toLong)) check(DayTimeIntervalType(), (-10 until 10).map(_ * 1000.toLong))
} }

View file

@ -121,7 +121,7 @@ private[hive] class SparkExecuteStatementOperation(
false, false,
timeFormatters) timeFormatters)
case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] | case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] |
YearMonthIntervalType | _: DayTimeIntervalType => _: YearMonthIntervalType | _: DayTimeIntervalType =>
to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters) to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters)
} }
} }
@ -377,7 +377,7 @@ object SparkExecuteStatementOperation {
val attrTypeString = field.dataType match { val attrTypeString = field.dataType match {
case NullType => "void" case NullType => "void"
case CalendarIntervalType => StringType.catalogString case CalendarIntervalType => StringType.catalogString
case YearMonthIntervalType => "interval_year_month" case _: YearMonthIntervalType => "interval_year_month"
case _: DayTimeIntervalType => "interval_day_time" case _: DayTimeIntervalType => "interval_day_time"
case other => other.catalogString case other => other.catalogString
} }

View file

@ -131,7 +131,8 @@ private[hive] class SparkGetColumnsOperation(
*/ */
private def getColumnSize(typ: DataType): Option[Int] = typ match { private def getColumnSize(typ: DataType): Option[Int] = typ match {
case dt @ (BooleanType | _: NumericType | DateType | TimestampType | case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
CalendarIntervalType | NullType | YearMonthIntervalType | _: DayTimeIntervalType) => CalendarIntervalType | NullType |
_: YearMonthIntervalType | _: DayTimeIntervalType) =>
Some(dt.defaultSize) Some(dt.defaultSize)
case CharType(n) => Some(n) case CharType(n) => Some(n)
case StructType(fields) => case StructType(fields) =>
@ -186,7 +187,7 @@ private[hive] class SparkGetColumnsOperation(
case _: MapType => java.sql.Types.JAVA_OBJECT case _: MapType => java.sql.Types.JAVA_OBJECT
case _: StructType => java.sql.Types.STRUCT case _: StructType => java.sql.Types.STRUCT
// Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER // Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
case _: CalendarIntervalType | YearMonthIntervalType | _: DayTimeIntervalType => case _: CalendarIntervalType | _: YearMonthIntervalType | _: DayTimeIntervalType =>
java.sql.Types.OTHER java.sql.Types.OTHER
case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}") case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
} }

View file

@ -392,8 +392,8 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase {
assert(rowSet.getString("TABLE_NAME") === viewName1) assert(rowSet.getString("TABLE_NAME") === viewName1)
assert(rowSet.getString("COLUMN_NAME") === "i") assert(rowSet.getString("COLUMN_NAME") === "i")
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER) assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql)) assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType().sql))
assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize) assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType().defaultSize)
assert(rowSet.getInt("DECIMAL_DIGITS") === 0) assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
assert(rowSet.getInt("NUM_PREC_RADIX") === 0) assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
assert(rowSet.getInt("NULLABLE") === 0) assert(rowSet.getInt("NULLABLE") === 0)

View file

@ -842,7 +842,7 @@ private[hive] trait HiveInspectors {
case TimestampType => PrimitiveObjectInspectorFactory.javaTimestampObjectInspector case TimestampType => PrimitiveObjectInspectorFactory.javaTimestampObjectInspector
case _: DayTimeIntervalType => case _: DayTimeIntervalType =>
PrimitiveObjectInspectorFactory.javaHiveIntervalDayTimeObjectInspector PrimitiveObjectInspectorFactory.javaHiveIntervalDayTimeObjectInspector
case YearMonthIntervalType => case _: YearMonthIntervalType =>
PrimitiveObjectInspectorFactory.javaHiveIntervalYearMonthObjectInspector PrimitiveObjectInspectorFactory.javaHiveIntervalYearMonthObjectInspector
// TODO decimal precision? // TODO decimal precision?
case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector
@ -891,7 +891,7 @@ private[hive] trait HiveInspectors {
getPrimitiveNullWritableConstantObjectInspector getPrimitiveNullWritableConstantObjectInspector
case Literal(_, _: DayTimeIntervalType) => case Literal(_, _: DayTimeIntervalType) =>
getHiveIntervalDayTimeWritableConstantObjectInspector getHiveIntervalDayTimeWritableConstantObjectInspector
case Literal(_, YearMonthIntervalType) => case Literal(_, _: YearMonthIntervalType) =>
getHiveIntervalYearMonthWritableConstantObjectInspector getHiveIntervalYearMonthWritableConstantObjectInspector
case Literal(value, ArrayType(dt, _)) => case Literal(value, ArrayType(dt, _)) =>
val listObjectInspector = toInspector(dt) val listObjectInspector = toInspector(dt)
@ -971,8 +971,8 @@ private[hive] trait HiveInspectors {
case _: JavaTimestampObjectInspector => TimestampType case _: JavaTimestampObjectInspector => TimestampType
case _: WritableHiveIntervalDayTimeObjectInspector => DayTimeIntervalType() case _: WritableHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
case _: JavaHiveIntervalDayTimeObjectInspector => DayTimeIntervalType() case _: JavaHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
case _: WritableHiveIntervalYearMonthObjectInspector => YearMonthIntervalType case _: WritableHiveIntervalYearMonthObjectInspector => YearMonthIntervalType()
case _: JavaHiveIntervalYearMonthObjectInspector => YearMonthIntervalType case _: JavaHiveIntervalYearMonthObjectInspector => YearMonthIntervalType()
case _: WritableVoidObjectInspector => NullType case _: WritableVoidObjectInspector => NullType
case _: JavaVoidObjectInspector => NullType case _: JavaVoidObjectInspector => NullType
} }
@ -1156,7 +1156,7 @@ private[hive] trait HiveInspectors {
case TimestampType => timestampTypeInfo case TimestampType => timestampTypeInfo
case NullType => voidTypeInfo case NullType => voidTypeInfo
case _: DayTimeIntervalType => intervalDayTimeTypeInfo case _: DayTimeIntervalType => intervalDayTimeTypeInfo
case YearMonthIntervalType => intervalYearMonthTypeInfo case _: YearMonthIntervalType => intervalYearMonthTypeInfo
case dt => case dt =>
throw new AnalysisException( throw new AnalysisException(
s"${dt.catalogString} cannot be converted to Hive TypeInfo") s"${dt.catalogString} cannot be converted to Hive TypeInfo")

View file

@ -546,7 +546,8 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
AttributeReference("a", DayTimeIntervalType())(), AttributeReference("a", DayTimeIntervalType())(),
AttributeReference("b", DayTimeIntervalType())(), AttributeReference("b", DayTimeIntervalType())(),
AttributeReference("c", DayTimeIntervalType())(), AttributeReference("c", DayTimeIntervalType())(),
AttributeReference("d", YearMonthIntervalType)()), // TODO(SPARK-35772): Check all year-month interval types in HiveInspectors tests
AttributeReference("d", YearMonthIntervalType())()),
child = child, child = child,
ioschema = hiveIOSchema), ioschema = hiveIOSchema),
df.select($"a", $"b", $"c", $"d").collect()) df.select($"a", $"b", $"c", $"d").collect())