[SPARK-35680][SQL] Add fields to YearMonthIntervalType
### What changes were proposed in this pull request? Extend `YearMonthIntervalType` to support interval fields. Valid interval field values: - 0 (YEAR) - 1 (MONTH) After the changes, the following year-month interval types are supported: 1. `YearMonthIntervalType(0, 0)` or `YearMonthIntervalType(YEAR, YEAR)` 2. `YearMonthIntervalType(0, 1)` or `YearMonthIntervalType(YEAR, MONTH)`. **It is the default one**. 3. `YearMonthIntervalType(1, 1)` or `YearMonthIntervalType(MONTH, MONTH)` Closes #32825 ### Why are the changes needed? In the current implementation, Spark supports only `interval year to month` but the SQL standard allows to specify the start and end fields. The changes will allow to follow ANSI SQL standard more precisely. ### Does this PR introduce _any_ user-facing change? Yes but `YearMonthIntervalType` has not been released yet. ### How was this patch tested? By existing test suites. Closes #32909 from MaxGekk/add-fields-to-YearMonthIntervalType. Authored-by: Max Gekk <max.gekk@gmail.com> Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
1012967ace
commit
61ce8f7649
|
@ -76,7 +76,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
|
||||||
*/
|
*/
|
||||||
public static final Set<DataType> mutableFieldTypes;
|
public static final Set<DataType> mutableFieldTypes;
|
||||||
|
|
||||||
// DecimalType and DayTimeIntervalType are also mutable
|
// DecimalType, DayTimeIntervalType and YearMonthIntervalType are also mutable
|
||||||
static {
|
static {
|
||||||
mutableFieldTypes = Collections.unmodifiableSet(
|
mutableFieldTypes = Collections.unmodifiableSet(
|
||||||
new HashSet<>(
|
new HashSet<>(
|
||||||
|
@ -90,8 +90,7 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
|
||||||
FloatType,
|
FloatType,
|
||||||
DoubleType,
|
DoubleType,
|
||||||
DateType,
|
DateType,
|
||||||
TimestampType,
|
TimestampType
|
||||||
YearMonthIntervalType
|
|
||||||
})));
|
})));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,7 +102,8 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
|
||||||
if (dt instanceof DecimalType) {
|
if (dt instanceof DecimalType) {
|
||||||
return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
|
return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
|
||||||
} else {
|
} else {
|
||||||
return dt instanceof DayTimeIntervalType || mutableFieldTypes.contains(dt);
|
return dt instanceof DayTimeIntervalType || dt instanceof YearMonthIntervalType ||
|
||||||
|
mutableFieldTypes.contains(dt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,7 +113,8 @@ public final class UnsafeRow extends InternalRow implements Externalizable, Kryo
|
||||||
}
|
}
|
||||||
|
|
||||||
return mutableFieldTypes.contains(dt) || dt instanceof DecimalType ||
|
return mutableFieldTypes.contains(dt) || dt instanceof DecimalType ||
|
||||||
dt instanceof CalendarIntervalType || dt instanceof DayTimeIntervalType;
|
dt instanceof CalendarIntervalType || dt instanceof DayTimeIntervalType ||
|
||||||
|
dt instanceof YearMonthIntervalType;
|
||||||
}
|
}
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
|
@ -99,11 +99,6 @@ public class DataTypes {
|
||||||
*/
|
*/
|
||||||
public static final DataType NullType = NullType$.MODULE$;
|
public static final DataType NullType = NullType$.MODULE$;
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets the YearMonthIntervalType object.
|
|
||||||
*/
|
|
||||||
public static final DataType YearMonthIntervalType = YearMonthIntervalType$.MODULE$;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an ArrayType by specifying the data type of elements ({@code elementType}).
|
* Creates an ArrayType by specifying the data type of elements ({@code elementType}).
|
||||||
* The field of {@code containsNull} is set to {@code true}.
|
* The field of {@code containsNull} is set to {@code true}.
|
||||||
|
@ -154,6 +149,20 @@ public class DataTypes {
|
||||||
return DayTimeIntervalType$.MODULE$.DEFAULT();
|
return DayTimeIntervalType$.MODULE$.DEFAULT();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a YearMonthIntervalType by specifying the start and end fields.
|
||||||
|
*/
|
||||||
|
public static YearMonthIntervalType createYearMonthIntervalType(byte startField, byte endField) {
|
||||||
|
return YearMonthIntervalType$.MODULE$.apply(startField, endField);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a YearMonthIntervalType with default start and end fields: interval year to month.
|
||||||
|
*/
|
||||||
|
public static YearMonthIntervalType createYearMonthIntervalType() {
|
||||||
|
return YearMonthIntervalType$.MODULE$.DEFAULT();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a MapType by specifying the data type of keys ({@code keyType}) and values
|
* Creates a MapType by specifying the data type of keys ({@code keyType}) and values
|
||||||
* ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}.
|
* ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}.
|
||||||
|
|
|
@ -77,7 +77,8 @@ object CatalystTypeConverters {
|
||||||
case DoubleType => DoubleConverter
|
case DoubleType => DoubleConverter
|
||||||
// TODO(SPARK-35726): Truncate java.time.Duration by fields of day-time interval type
|
// TODO(SPARK-35726): Truncate java.time.Duration by fields of day-time interval type
|
||||||
case _: DayTimeIntervalType => DurationConverter
|
case _: DayTimeIntervalType => DurationConverter
|
||||||
case YearMonthIntervalType => PeriodConverter
|
// TODO(SPARK-35769): Truncate java.time.Period by fields of year-month interval type
|
||||||
|
case _: YearMonthIntervalType => PeriodConverter
|
||||||
case dataType: DataType => IdentityConverter(dataType)
|
case dataType: DataType => IdentityConverter(dataType)
|
||||||
}
|
}
|
||||||
converter.asInstanceOf[CatalystTypeConverter[Any, Any, Any]]
|
converter.asInstanceOf[CatalystTypeConverter[Any, Any, Any]]
|
||||||
|
|
|
@ -132,7 +132,7 @@ object InternalRow {
|
||||||
case BooleanType => (input, ordinal) => input.getBoolean(ordinal)
|
case BooleanType => (input, ordinal) => input.getBoolean(ordinal)
|
||||||
case ByteType => (input, ordinal) => input.getByte(ordinal)
|
case ByteType => (input, ordinal) => input.getByte(ordinal)
|
||||||
case ShortType => (input, ordinal) => input.getShort(ordinal)
|
case ShortType => (input, ordinal) => input.getShort(ordinal)
|
||||||
case IntegerType | DateType | YearMonthIntervalType =>
|
case IntegerType | DateType | _: YearMonthIntervalType =>
|
||||||
(input, ordinal) => input.getInt(ordinal)
|
(input, ordinal) => input.getInt(ordinal)
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
(input, ordinal) => input.getLong(ordinal)
|
(input, ordinal) => input.getLong(ordinal)
|
||||||
|
@ -169,7 +169,7 @@ object InternalRow {
|
||||||
case BooleanType => (input, v) => input.setBoolean(ordinal, v.asInstanceOf[Boolean])
|
case BooleanType => (input, v) => input.setBoolean(ordinal, v.asInstanceOf[Boolean])
|
||||||
case ByteType => (input, v) => input.setByte(ordinal, v.asInstanceOf[Byte])
|
case ByteType => (input, v) => input.setByte(ordinal, v.asInstanceOf[Byte])
|
||||||
case ShortType => (input, v) => input.setShort(ordinal, v.asInstanceOf[Short])
|
case ShortType => (input, v) => input.setShort(ordinal, v.asInstanceOf[Short])
|
||||||
case IntegerType | DateType | YearMonthIntervalType =>
|
case IntegerType | DateType | _: YearMonthIntervalType =>
|
||||||
(input, v) => input.setInt(ordinal, v.asInstanceOf[Int])
|
(input, v) => input.setInt(ordinal, v.asInstanceOf[Int])
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
(input, v) => input.setLong(ordinal, v.asInstanceOf[Long])
|
(input, v) => input.setLong(ordinal, v.asInstanceOf[Long])
|
||||||
|
|
|
@ -121,7 +121,7 @@ object JavaTypeInference {
|
||||||
case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true)
|
case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true)
|
||||||
case c: Class[_] if c == classOf[java.time.LocalDateTime] => (TimestampWithoutTZType, true)
|
case c: Class[_] if c == classOf[java.time.LocalDateTime] => (TimestampWithoutTZType, true)
|
||||||
case c: Class[_] if c == classOf[java.time.Duration] => (DayTimeIntervalType(), true)
|
case c: Class[_] if c == classOf[java.time.Duration] => (DayTimeIntervalType(), true)
|
||||||
case c: Class[_] if c == classOf[java.time.Period] => (YearMonthIntervalType, true)
|
case c: Class[_] if c == classOf[java.time.Period] => (YearMonthIntervalType(), true)
|
||||||
|
|
||||||
case _ if typeToken.isArray =>
|
case _ if typeToken.isArray =>
|
||||||
val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet)
|
val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet)
|
||||||
|
|
|
@ -761,7 +761,7 @@ object ScalaReflection extends ScalaReflection {
|
||||||
case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
|
case t if isSubtype(t, localTypeOf[java.time.Duration]) =>
|
||||||
Schema(DayTimeIntervalType(), nullable = true)
|
Schema(DayTimeIntervalType(), nullable = true)
|
||||||
case t if isSubtype(t, localTypeOf[java.time.Period]) =>
|
case t if isSubtype(t, localTypeOf[java.time.Period]) =>
|
||||||
Schema(YearMonthIntervalType, nullable = true)
|
Schema(YearMonthIntervalType(), nullable = true)
|
||||||
case t if isSubtype(t, localTypeOf[BigDecimal]) =>
|
case t if isSubtype(t, localTypeOf[BigDecimal]) =>
|
||||||
Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
|
Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
|
||||||
case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
|
case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
|
||||||
|
@ -860,8 +860,7 @@ object ScalaReflection extends ScalaReflection {
|
||||||
TimestampType -> classOf[TimestampType.InternalType],
|
TimestampType -> classOf[TimestampType.InternalType],
|
||||||
TimestampWithoutTZType -> classOf[TimestampWithoutTZType.InternalType],
|
TimestampWithoutTZType -> classOf[TimestampWithoutTZType.InternalType],
|
||||||
BinaryType -> classOf[BinaryType.InternalType],
|
BinaryType -> classOf[BinaryType.InternalType],
|
||||||
CalendarIntervalType -> classOf[CalendarInterval],
|
CalendarIntervalType -> classOf[CalendarInterval]
|
||||||
YearMonthIntervalType -> classOf[YearMonthIntervalType.InternalType]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
val typeBoxedJavaMapping = Map[DataType, Class[_]](
|
val typeBoxedJavaMapping = Map[DataType, Class[_]](
|
||||||
|
@ -874,14 +873,14 @@ object ScalaReflection extends ScalaReflection {
|
||||||
DoubleType -> classOf[java.lang.Double],
|
DoubleType -> classOf[java.lang.Double],
|
||||||
DateType -> classOf[java.lang.Integer],
|
DateType -> classOf[java.lang.Integer],
|
||||||
TimestampType -> classOf[java.lang.Long],
|
TimestampType -> classOf[java.lang.Long],
|
||||||
TimestampWithoutTZType -> classOf[java.lang.Long],
|
TimestampWithoutTZType -> classOf[java.lang.Long]
|
||||||
YearMonthIntervalType -> classOf[java.lang.Integer]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def dataTypeJavaClass(dt: DataType): Class[_] = {
|
def dataTypeJavaClass(dt: DataType): Class[_] = {
|
||||||
dt match {
|
dt match {
|
||||||
case _: DecimalType => classOf[Decimal]
|
case _: DecimalType => classOf[Decimal]
|
||||||
case it: DayTimeIntervalType => classOf[it.InternalType]
|
case it: DayTimeIntervalType => classOf[it.InternalType]
|
||||||
|
case it: YearMonthIntervalType => classOf[it.InternalType]
|
||||||
case _: StructType => classOf[InternalRow]
|
case _: StructType => classOf[InternalRow]
|
||||||
case _: ArrayType => classOf[ArrayData]
|
case _: ArrayType => classOf[ArrayData]
|
||||||
case _: MapType => classOf[MapData]
|
case _: MapType => classOf[MapData]
|
||||||
|
@ -893,6 +892,7 @@ object ScalaReflection extends ScalaReflection {
|
||||||
def javaBoxedType(dt: DataType): Class[_] = dt match {
|
def javaBoxedType(dt: DataType): Class[_] = dt match {
|
||||||
case _: DecimalType => classOf[Decimal]
|
case _: DecimalType => classOf[Decimal]
|
||||||
case _: DayTimeIntervalType => classOf[java.lang.Long]
|
case _: DayTimeIntervalType => classOf[java.lang.Long]
|
||||||
|
case _: YearMonthIntervalType => classOf[java.lang.Integer]
|
||||||
case BinaryType => classOf[Array[Byte]]
|
case BinaryType => classOf[Array[Byte]]
|
||||||
case StringType => classOf[UTF8String]
|
case StringType => classOf[UTF8String]
|
||||||
case CalendarIntervalType => classOf[CalendarInterval]
|
case CalendarIntervalType => classOf[CalendarInterval]
|
||||||
|
|
|
@ -125,7 +125,7 @@ object SerializerBuildHelper {
|
||||||
def createSerializerForJavaPeriod(inputObject: Expression): Expression = {
|
def createSerializerForJavaPeriod(inputObject: Expression): Expression = {
|
||||||
StaticInvoke(
|
StaticInvoke(
|
||||||
IntervalUtils.getClass,
|
IntervalUtils.getClass,
|
||||||
YearMonthIntervalType,
|
YearMonthIntervalType(),
|
||||||
"periodToMonths",
|
"periodToMonths",
|
||||||
inputObject :: Nil,
|
inputObject :: Nil,
|
||||||
returnNullable = false)
|
returnNullable = false)
|
||||||
|
|
|
@ -351,10 +351,10 @@ class Analyzer(override val catalogManager: CatalogManager)
|
||||||
case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match {
|
case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match {
|
||||||
case (DateType, _: DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r)
|
case (DateType, _: DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r)
|
||||||
case (_: DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l)
|
case (_: DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l)
|
||||||
case (DateType, YearMonthIntervalType) => DateAddYMInterval(l, r)
|
case (DateType, _: YearMonthIntervalType) => DateAddYMInterval(l, r)
|
||||||
case (YearMonthIntervalType, DateType) => DateAddYMInterval(r, l)
|
case (_: YearMonthIntervalType, DateType) => DateAddYMInterval(r, l)
|
||||||
case (TimestampType, YearMonthIntervalType) => TimestampAddYMInterval(l, r)
|
case (TimestampType, _: YearMonthIntervalType) => TimestampAddYMInterval(l, r)
|
||||||
case (YearMonthIntervalType, TimestampType) => TimestampAddYMInterval(r, l)
|
case (_: YearMonthIntervalType, TimestampType) => TimestampAddYMInterval(r, l)
|
||||||
case (CalendarIntervalType, CalendarIntervalType) |
|
case (CalendarIntervalType, CalendarIntervalType) |
|
||||||
(_: DayTimeIntervalType, _: DayTimeIntervalType) => a
|
(_: DayTimeIntervalType, _: DayTimeIntervalType) => a
|
||||||
case (DateType, CalendarIntervalType) => DateAddInterval(l, r, ansiEnabled = f)
|
case (DateType, CalendarIntervalType) => DateAddInterval(l, r, ansiEnabled = f)
|
||||||
|
@ -368,9 +368,9 @@ class Analyzer(override val catalogManager: CatalogManager)
|
||||||
case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match {
|
case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match {
|
||||||
case (DateType, _: DayTimeIntervalType) =>
|
case (DateType, _: DayTimeIntervalType) =>
|
||||||
DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f)))
|
DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f)))
|
||||||
case (DateType, YearMonthIntervalType) =>
|
case (DateType, _: YearMonthIntervalType) =>
|
||||||
DatetimeSub(l, r, DateAddYMInterval(l, UnaryMinus(r, f)))
|
DatetimeSub(l, r, DateAddYMInterval(l, UnaryMinus(r, f)))
|
||||||
case (TimestampType, YearMonthIntervalType) =>
|
case (TimestampType, _: YearMonthIntervalType) =>
|
||||||
DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, f)))
|
DatetimeSub(l, r, TimestampAddYMInterval(l, UnaryMinus(r, f)))
|
||||||
case (CalendarIntervalType, CalendarIntervalType) |
|
case (CalendarIntervalType, CalendarIntervalType) |
|
||||||
(_: DayTimeIntervalType, _: DayTimeIntervalType) => s
|
(_: DayTimeIntervalType, _: DayTimeIntervalType) => s
|
||||||
|
@ -387,15 +387,15 @@ class Analyzer(override val catalogManager: CatalogManager)
|
||||||
case m @ Multiply(l, r, f) if m.childrenResolved => (l.dataType, r.dataType) match {
|
case m @ Multiply(l, r, f) if m.childrenResolved => (l.dataType, r.dataType) match {
|
||||||
case (CalendarIntervalType, _) => MultiplyInterval(l, r, f)
|
case (CalendarIntervalType, _) => MultiplyInterval(l, r, f)
|
||||||
case (_, CalendarIntervalType) => MultiplyInterval(r, l, f)
|
case (_, CalendarIntervalType) => MultiplyInterval(r, l, f)
|
||||||
case (YearMonthIntervalType, _) => MultiplyYMInterval(l, r)
|
case (_: YearMonthIntervalType, _) => MultiplyYMInterval(l, r)
|
||||||
case (_, YearMonthIntervalType) => MultiplyYMInterval(r, l)
|
case (_, _: YearMonthIntervalType) => MultiplyYMInterval(r, l)
|
||||||
case (_: DayTimeIntervalType, _) => MultiplyDTInterval(l, r)
|
case (_: DayTimeIntervalType, _) => MultiplyDTInterval(l, r)
|
||||||
case (_, _: DayTimeIntervalType) => MultiplyDTInterval(r, l)
|
case (_, _: DayTimeIntervalType) => MultiplyDTInterval(r, l)
|
||||||
case _ => m
|
case _ => m
|
||||||
}
|
}
|
||||||
case d @ Divide(l, r, f) if d.childrenResolved => (l.dataType, r.dataType) match {
|
case d @ Divide(l, r, f) if d.childrenResolved => (l.dataType, r.dataType) match {
|
||||||
case (CalendarIntervalType, _) => DivideInterval(l, r, f)
|
case (CalendarIntervalType, _) => DivideInterval(l, r, f)
|
||||||
case (YearMonthIntervalType, _) => DivideYMInterval(l, r)
|
case (_: YearMonthIntervalType, _) => DivideYMInterval(l, r)
|
||||||
case (_: DayTimeIntervalType, _) => DivideDTInterval(l, r)
|
case (_: DayTimeIntervalType, _) => DivideDTInterval(l, r)
|
||||||
case _ => d
|
case _ => d
|
||||||
}
|
}
|
||||||
|
|
|
@ -310,8 +310,11 @@ package object dsl {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Creates a new AttributeReference of the year-month interval type */
|
/** Creates a new AttributeReference of the year-month interval type */
|
||||||
def yearMonthInterval: AttributeReference = {
|
def yearMonthInterval(startField: Byte, endField: Byte): AttributeReference = {
|
||||||
AttributeReference(s, YearMonthIntervalType, nullable = true)()
|
AttributeReference(s, YearMonthIntervalType(startField, endField), nullable = true)()
|
||||||
|
}
|
||||||
|
def yearMonthInterval(): AttributeReference = {
|
||||||
|
AttributeReference(s, YearMonthIntervalType(), nullable = true)()
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Creates a new AttributeReference of type binary */
|
/** Creates a new AttributeReference of type binary */
|
||||||
|
|
|
@ -116,7 +116,7 @@ object RowEncoder {
|
||||||
|
|
||||||
case _: DayTimeIntervalType => createSerializerForJavaDuration(inputObject)
|
case _: DayTimeIntervalType => createSerializerForJavaDuration(inputObject)
|
||||||
|
|
||||||
case YearMonthIntervalType => createSerializerForJavaPeriod(inputObject)
|
case _: YearMonthIntervalType => createSerializerForJavaPeriod(inputObject)
|
||||||
|
|
||||||
case d: DecimalType =>
|
case d: DecimalType =>
|
||||||
CheckOverflow(StaticInvoke(
|
CheckOverflow(StaticInvoke(
|
||||||
|
@ -239,7 +239,7 @@ object RowEncoder {
|
||||||
ObjectType(classOf[java.sql.Date])
|
ObjectType(classOf[java.sql.Date])
|
||||||
}
|
}
|
||||||
case _: DayTimeIntervalType => ObjectType(classOf[java.time.Duration])
|
case _: DayTimeIntervalType => ObjectType(classOf[java.time.Duration])
|
||||||
case YearMonthIntervalType => ObjectType(classOf[java.time.Period])
|
case _: YearMonthIntervalType => ObjectType(classOf[java.time.Period])
|
||||||
case _: DecimalType => ObjectType(classOf[java.math.BigDecimal])
|
case _: DecimalType => ObjectType(classOf[java.math.BigDecimal])
|
||||||
case StringType => ObjectType(classOf[java.lang.String])
|
case StringType => ObjectType(classOf[java.lang.String])
|
||||||
case _: ArrayType => ObjectType(classOf[scala.collection.Seq[_]])
|
case _: ArrayType => ObjectType(classOf[scala.collection.Seq[_]])
|
||||||
|
@ -299,7 +299,7 @@ object RowEncoder {
|
||||||
|
|
||||||
case _: DayTimeIntervalType => createDeserializerForDuration(input)
|
case _: DayTimeIntervalType => createDeserializerForDuration(input)
|
||||||
|
|
||||||
case YearMonthIntervalType => createDeserializerForPeriod(input)
|
case _: YearMonthIntervalType => createDeserializerForPeriod(input)
|
||||||
|
|
||||||
case _: DecimalType => createDeserializerForJavaBigDecimal(input, returnNullable = false)
|
case _: DecimalType => createDeserializerForJavaBigDecimal(input, returnNullable = false)
|
||||||
|
|
||||||
|
|
|
@ -79,7 +79,7 @@ object Cast {
|
||||||
|
|
||||||
case (StringType, CalendarIntervalType) => true
|
case (StringType, CalendarIntervalType) => true
|
||||||
case (StringType, _: DayTimeIntervalType) => true
|
case (StringType, _: DayTimeIntervalType) => true
|
||||||
case (StringType, YearMonthIntervalType) => true
|
case (StringType, _: YearMonthIntervalType) => true
|
||||||
|
|
||||||
case (StringType, _: NumericType) => true
|
case (StringType, _: NumericType) => true
|
||||||
case (BooleanType, _: NumericType) => true
|
case (BooleanType, _: NumericType) => true
|
||||||
|
@ -422,9 +422,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
case pudt: PythonUserDefinedType => castToString(pudt.sqlType)
|
case pudt: PythonUserDefinedType => castToString(pudt.sqlType)
|
||||||
case udt: UserDefinedType[_] =>
|
case udt: UserDefinedType[_] =>
|
||||||
buildCast[Any](_, o => UTF8String.fromString(udt.deserialize(o).toString))
|
buildCast[Any](_, o => UTF8String.fromString(udt.deserialize(o).toString))
|
||||||
case YearMonthIntervalType =>
|
case YearMonthIntervalType(startField, endField) =>
|
||||||
buildCast[Int](_, i => UTF8String.fromString(
|
buildCast[Int](_, i => UTF8String.fromString(
|
||||||
IntervalUtils.toYearMonthIntervalString(i, ANSI_STYLE)))
|
IntervalUtils.toYearMonthIntervalString(i, ANSI_STYLE, startField, endField)))
|
||||||
case DayTimeIntervalType(startField, endField) =>
|
case DayTimeIntervalType(startField, endField) =>
|
||||||
buildCast[Long](_, i => UTF8String.fromString(
|
buildCast[Long](_, i => UTF8String.fromString(
|
||||||
IntervalUtils.toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)))
|
IntervalUtils.toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)))
|
||||||
|
@ -566,8 +566,11 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
IntervalUtils.castStringToDTInterval(s, it.startField, it.endField))
|
IntervalUtils.castStringToDTInterval(s, it.startField, it.endField))
|
||||||
}
|
}
|
||||||
|
|
||||||
private[this] def castToYearMonthInterval(from: DataType): Any => Any = from match {
|
private[this] def castToYearMonthInterval(
|
||||||
case StringType => buildCast[UTF8String](_, s => IntervalUtils.castStringToYMInterval(s))
|
from: DataType,
|
||||||
|
it: YearMonthIntervalType): Any => Any = from match {
|
||||||
|
case StringType => buildCast[UTF8String](_, s =>
|
||||||
|
IntervalUtils.castStringToYMInterval(s, it.startField, it.endField))
|
||||||
}
|
}
|
||||||
|
|
||||||
// LongConverter
|
// LongConverter
|
||||||
|
@ -876,7 +879,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
case TimestampWithoutTZType => castToTimestampWithoutTZ(from)
|
case TimestampWithoutTZType => castToTimestampWithoutTZ(from)
|
||||||
case CalendarIntervalType => castToInterval(from)
|
case CalendarIntervalType => castToInterval(from)
|
||||||
case it: DayTimeIntervalType => castToDayTimeInterval(from, it)
|
case it: DayTimeIntervalType => castToDayTimeInterval(from, it)
|
||||||
case YearMonthIntervalType => castToYearMonthInterval(from)
|
case it: YearMonthIntervalType => castToYearMonthInterval(from, it)
|
||||||
case BooleanType => castToBoolean(from)
|
case BooleanType => castToBoolean(from)
|
||||||
case ByteType => castToByte(from)
|
case ByteType => castToByte(from)
|
||||||
case ShortType => castToShort(from)
|
case ShortType => castToShort(from)
|
||||||
|
@ -937,7 +940,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
case TimestampWithoutTZType => castToTimestampWithoutTZCode(from, ctx)
|
case TimestampWithoutTZType => castToTimestampWithoutTZCode(from, ctx)
|
||||||
case CalendarIntervalType => castToIntervalCode(from)
|
case CalendarIntervalType => castToIntervalCode(from)
|
||||||
case it: DayTimeIntervalType => castToDayTimeIntervalCode(from, it)
|
case it: DayTimeIntervalType => castToDayTimeIntervalCode(from, it)
|
||||||
case YearMonthIntervalType => castToYearMonthIntervalCode(from)
|
case it: YearMonthIntervalType => castToYearMonthIntervalCode(from, it)
|
||||||
case BooleanType => castToBooleanCode(from)
|
case BooleanType => castToBooleanCode(from)
|
||||||
case ByteType => castToByteCode(from, ctx)
|
case ByteType => castToByteCode(from, ctx)
|
||||||
case ShortType => castToShortCode(from, ctx)
|
case ShortType => castToShortCode(from, ctx)
|
||||||
|
@ -1176,15 +1179,16 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
(c, evPrim, evNull) => {
|
(c, evPrim, evNull) => {
|
||||||
code"$evPrim = UTF8String.fromString($udtRef.deserialize($c).toString());"
|
code"$evPrim = UTF8String.fromString($udtRef.deserialize($c).toString());"
|
||||||
}
|
}
|
||||||
case YearMonthIntervalType =>
|
case i: YearMonthIntervalType =>
|
||||||
val iu = IntervalUtils.getClass.getName.stripSuffix("$")
|
val iu = IntervalUtils.getClass.getName.stripSuffix("$")
|
||||||
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
|
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
|
||||||
val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
|
val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
|
||||||
(c, evPrim, _) =>
|
(c, evPrim, _) =>
|
||||||
code"""
|
code"""
|
||||||
$evPrim = UTF8String.fromString($iu.toYearMonthIntervalString($c, $style));
|
$evPrim = UTF8String.fromString($iu.toYearMonthIntervalString($c, $style,
|
||||||
|
(byte)${i.startField}, (byte)${i.endField}));
|
||||||
"""
|
"""
|
||||||
case i : DayTimeIntervalType =>
|
case i: DayTimeIntervalType =>
|
||||||
val iu = IntervalUtils.getClass.getName.stripSuffix("$")
|
val iu = IntervalUtils.getClass.getName.stripSuffix("$")
|
||||||
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
|
val iss = IntervalStringStyles.getClass.getName.stripSuffix("$")
|
||||||
val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
|
val style = s"$iss$$.MODULE$$.ANSI_STYLE()"
|
||||||
|
@ -1441,10 +1445,15 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
|
|
||||||
private[this] def castToYearMonthIntervalCode(from: DataType): CastFunction = from match {
|
private[this] def castToYearMonthIntervalCode(
|
||||||
|
from: DataType,
|
||||||
|
it: YearMonthIntervalType): CastFunction = from match {
|
||||||
case StringType =>
|
case StringType =>
|
||||||
val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
val util = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
||||||
(c, evPrim, _) => code"$evPrim = $util.castStringToYMInterval($c);"
|
(c, evPrim, _) =>
|
||||||
|
code"""
|
||||||
|
$evPrim = $util.castStringToYMInterval($c, (byte)${it.startField}, (byte)${it.endField});
|
||||||
|
"""
|
||||||
}
|
}
|
||||||
|
|
||||||
private[this] def decimalToTimestampCode(d: ExprValue): Block = {
|
private[this] def decimalToTimestampCode(d: ExprValue): Block = {
|
||||||
|
@ -2012,7 +2021,7 @@ object AnsiCast {
|
||||||
|
|
||||||
case (StringType, _: CalendarIntervalType) => true
|
case (StringType, _: CalendarIntervalType) => true
|
||||||
case (StringType, _: DayTimeIntervalType) => true
|
case (StringType, _: DayTimeIntervalType) => true
|
||||||
case (StringType, YearMonthIntervalType) => true
|
case (StringType, _: YearMonthIntervalType) => true
|
||||||
|
|
||||||
case (StringType, DateType) => true
|
case (StringType, DateType) => true
|
||||||
case (TimestampType, DateType) => true
|
case (TimestampType, DateType) => true
|
||||||
|
|
|
@ -157,7 +157,7 @@ object InterpretedUnsafeProjection {
|
||||||
case ShortType =>
|
case ShortType =>
|
||||||
(v, i) => writer.write(i, v.getShort(i))
|
(v, i) => writer.write(i, v.getShort(i))
|
||||||
|
|
||||||
case IntegerType | DateType | YearMonthIntervalType =>
|
case IntegerType | DateType | _: YearMonthIntervalType =>
|
||||||
(v, i) => writer.write(i, v.getInt(i))
|
(v, i) => writer.write(i, v.getInt(i))
|
||||||
|
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
|
|
|
@ -194,7 +194,7 @@ final class SpecificInternalRow(val values: Array[MutableValue]) extends BaseGen
|
||||||
|
|
||||||
private[this] def dataTypeToMutableValue(dataType: DataType): MutableValue = dataType match {
|
private[this] def dataTypeToMutableValue(dataType: DataType): MutableValue = dataType match {
|
||||||
// We use INT for DATE and YearMonthIntervalType internally
|
// We use INT for DATE and YearMonthIntervalType internally
|
||||||
case IntegerType | DateType | YearMonthIntervalType => new MutableInt
|
case IntegerType | DateType | _: YearMonthIntervalType => new MutableInt
|
||||||
// We use Long for Timestamp, Timestamp without time zone and DayTimeInterval internally
|
// We use Long for Timestamp, Timestamp without time zone and DayTimeInterval internally
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
new MutableLong
|
new MutableLong
|
||||||
|
|
|
@ -57,14 +57,14 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
|
||||||
private lazy val resultType = child.dataType match {
|
private lazy val resultType = child.dataType match {
|
||||||
case DecimalType.Fixed(p, s) =>
|
case DecimalType.Fixed(p, s) =>
|
||||||
DecimalType.bounded(p + 4, s + 4)
|
DecimalType.bounded(p + 4, s + 4)
|
||||||
case _: YearMonthIntervalType => YearMonthIntervalType
|
case _: YearMonthIntervalType => YearMonthIntervalType()
|
||||||
case _: DayTimeIntervalType => DayTimeIntervalType()
|
case _: DayTimeIntervalType => DayTimeIntervalType()
|
||||||
case _ => DoubleType
|
case _ => DoubleType
|
||||||
}
|
}
|
||||||
|
|
||||||
private lazy val sumDataType = child.dataType match {
|
private lazy val sumDataType = child.dataType match {
|
||||||
case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
|
case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
|
||||||
case _: YearMonthIntervalType => YearMonthIntervalType
|
case _: YearMonthIntervalType => YearMonthIntervalType()
|
||||||
case _: DayTimeIntervalType => DayTimeIntervalType()
|
case _: DayTimeIntervalType => DayTimeIntervalType()
|
||||||
case _ => DoubleType
|
case _ => DoubleType
|
||||||
}
|
}
|
||||||
|
@ -92,7 +92,7 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
|
||||||
Divide(sum, count.cast(DecimalType.LongDecimal), failOnError = false)).cast(resultType)
|
Divide(sum, count.cast(DecimalType.LongDecimal), failOnError = false)).cast(resultType)
|
||||||
case _: YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
If(EqualTo(count, Literal(0L)),
|
If(EqualTo(count, Literal(0L)),
|
||||||
Literal(null, YearMonthIntervalType), DivideYMInterval(sum, count))
|
Literal(null, YearMonthIntervalType()), DivideYMInterval(sum, count))
|
||||||
case _: DayTimeIntervalType =>
|
case _: DayTimeIntervalType =>
|
||||||
If(EqualTo(count, Literal(0L)),
|
If(EqualTo(count, Literal(0L)),
|
||||||
Literal(null, DayTimeIntervalType()), DivideDTInterval(sum, count))
|
Literal(null, DayTimeIntervalType()), DivideDTInterval(sum, count))
|
||||||
|
|
|
@ -59,7 +59,7 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
|
||||||
case DecimalType.Fixed(precision, scale) =>
|
case DecimalType.Fixed(precision, scale) =>
|
||||||
DecimalType.bounded(precision + 10, scale)
|
DecimalType.bounded(precision + 10, scale)
|
||||||
case _: IntegralType => LongType
|
case _: IntegralType => LongType
|
||||||
case _: YearMonthIntervalType => YearMonthIntervalType
|
case it: YearMonthIntervalType => it
|
||||||
case it: DayTimeIntervalType => it
|
case it: DayTimeIntervalType => it
|
||||||
case _ => DoubleType
|
case _ => DoubleType
|
||||||
}
|
}
|
||||||
|
|
|
@ -85,7 +85,7 @@ case class UnaryMinus(
|
||||||
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
||||||
val method = if (failOnError) "negateExact" else "negate"
|
val method = if (failOnError) "negateExact" else "negate"
|
||||||
defineCodeGen(ctx, ev, c => s"$iu.$method($c)")
|
defineCodeGen(ctx, ev, c => s"$iu.$method($c)")
|
||||||
case _: DayTimeIntervalType | YearMonthIntervalType =>
|
case _: DayTimeIntervalType | _: YearMonthIntervalType =>
|
||||||
nullSafeCodeGen(ctx, ev, eval => {
|
nullSafeCodeGen(ctx, ev, eval => {
|
||||||
val mathClass = classOf[Math].getName
|
val mathClass = classOf[Math].getName
|
||||||
s"${ev.value} = $mathClass.negateExact($eval);"
|
s"${ev.value} = $mathClass.negateExact($eval);"
|
||||||
|
@ -97,7 +97,7 @@ case class UnaryMinus(
|
||||||
IntervalUtils.negateExact(input.asInstanceOf[CalendarInterval])
|
IntervalUtils.negateExact(input.asInstanceOf[CalendarInterval])
|
||||||
case CalendarIntervalType => IntervalUtils.negate(input.asInstanceOf[CalendarInterval])
|
case CalendarIntervalType => IntervalUtils.negate(input.asInstanceOf[CalendarInterval])
|
||||||
case _: DayTimeIntervalType => Math.negateExact(input.asInstanceOf[Long])
|
case _: DayTimeIntervalType => Math.negateExact(input.asInstanceOf[Long])
|
||||||
case YearMonthIntervalType => Math.negateExact(input.asInstanceOf[Int])
|
case _: YearMonthIntervalType => Math.negateExact(input.asInstanceOf[Int])
|
||||||
case _ => numeric.negate(input)
|
case _ => numeric.negate(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -229,7 +229,7 @@ abstract class BinaryArithmetic extends BinaryOperator with NullIntolerant {
|
||||||
case CalendarIntervalType =>
|
case CalendarIntervalType =>
|
||||||
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
val iu = IntervalUtils.getClass.getCanonicalName.stripSuffix("$")
|
||||||
defineCodeGen(ctx, ev, (eval1, eval2) => s"$iu.$calendarIntervalMethod($eval1, $eval2)")
|
defineCodeGen(ctx, ev, (eval1, eval2) => s"$iu.$calendarIntervalMethod($eval1, $eval2)")
|
||||||
case _: DayTimeIntervalType | YearMonthIntervalType =>
|
case _: DayTimeIntervalType | _: YearMonthIntervalType =>
|
||||||
assert(exactMathMethod.isDefined,
|
assert(exactMathMethod.isDefined,
|
||||||
s"The expression '$nodeName' must override the exactMathMethod() method " +
|
s"The expression '$nodeName' must override the exactMathMethod() method " +
|
||||||
"if it is supposed to operate over interval types.")
|
"if it is supposed to operate over interval types.")
|
||||||
|
@ -319,7 +319,7 @@ case class Add(
|
||||||
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
|
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
|
||||||
case _: DayTimeIntervalType =>
|
case _: DayTimeIntervalType =>
|
||||||
Math.addExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
|
Math.addExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
Math.addExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
|
Math.addExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
|
||||||
case _ => numeric.plus(input1, input2)
|
case _ => numeric.plus(input1, input2)
|
||||||
}
|
}
|
||||||
|
@ -365,7 +365,7 @@ case class Subtract(
|
||||||
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
|
input1.asInstanceOf[CalendarInterval], input2.asInstanceOf[CalendarInterval])
|
||||||
case _: DayTimeIntervalType =>
|
case _: DayTimeIntervalType =>
|
||||||
Math.subtractExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
|
Math.subtractExact(input1.asInstanceOf[Long], input2.asInstanceOf[Long])
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
Math.subtractExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
|
Math.subtractExact(input1.asInstanceOf[Int], input2.asInstanceOf[Int])
|
||||||
case _ => numeric.minus(input1, input2)
|
case _ => numeric.minus(input1, input2)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1816,7 +1816,7 @@ object CodeGenerator extends Logging {
|
||||||
case BooleanType => JAVA_BOOLEAN
|
case BooleanType => JAVA_BOOLEAN
|
||||||
case ByteType => JAVA_BYTE
|
case ByteType => JAVA_BYTE
|
||||||
case ShortType => JAVA_SHORT
|
case ShortType => JAVA_SHORT
|
||||||
case IntegerType | DateType | YearMonthIntervalType => JAVA_INT
|
case IntegerType | DateType | _: YearMonthIntervalType => JAVA_INT
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => JAVA_LONG
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType => JAVA_LONG
|
||||||
case FloatType => JAVA_FLOAT
|
case FloatType => JAVA_FLOAT
|
||||||
case DoubleType => JAVA_DOUBLE
|
case DoubleType => JAVA_DOUBLE
|
||||||
|
@ -1837,7 +1837,7 @@ object CodeGenerator extends Logging {
|
||||||
case BooleanType => java.lang.Boolean.TYPE
|
case BooleanType => java.lang.Boolean.TYPE
|
||||||
case ByteType => java.lang.Byte.TYPE
|
case ByteType => java.lang.Byte.TYPE
|
||||||
case ShortType => java.lang.Short.TYPE
|
case ShortType => java.lang.Short.TYPE
|
||||||
case IntegerType | DateType | YearMonthIntervalType => java.lang.Integer.TYPE
|
case IntegerType | DateType | _: YearMonthIntervalType => java.lang.Integer.TYPE
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
java.lang.Long.TYPE
|
java.lang.Long.TYPE
|
||||||
case FloatType => java.lang.Float.TYPE
|
case FloatType => java.lang.Float.TYPE
|
||||||
|
|
|
@ -2564,14 +2564,14 @@ case class Sequence(
|
||||||
|1. The start and stop expressions must resolve to the same type.
|
|1. The start and stop expressions must resolve to the same type.
|
||||||
|2. If start and stop expressions resolve to the 'date' or 'timestamp' type
|
|2. If start and stop expressions resolve to the 'date' or 'timestamp' type
|
||||||
|then the step expression must resolve to the 'interval' or
|
|then the step expression must resolve to the 'interval' or
|
||||||
|'${YearMonthIntervalType.typeName}' or '${DayTimeIntervalType.simpleString}' type,
|
|'${YearMonthIntervalType.simpleString}' or '${DayTimeIntervalType.simpleString}' type,
|
||||||
|otherwise to the same type as the start and stop expressions.
|
|otherwise to the same type as the start and stop expressions.
|
||||||
""".stripMargin)
|
""".stripMargin)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def isNotIntervalType(expr: Expression) = expr.dataType match {
|
private def isNotIntervalType(expr: Expression) = expr.dataType match {
|
||||||
case CalendarIntervalType | YearMonthIntervalType | _: DayTimeIntervalType => false
|
case CalendarIntervalType | _: YearMonthIntervalType | _: DayTimeIntervalType => false
|
||||||
case _ => true
|
case _ => true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2749,10 +2749,10 @@ object Sequence {
|
||||||
|
|
||||||
override val defaultStep: DefaultStep = new DefaultStep(
|
override val defaultStep: DefaultStep = new DefaultStep(
|
||||||
(dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
|
(dt.ordering.lteq _).asInstanceOf[LessThanOrEqualFn],
|
||||||
YearMonthIntervalType,
|
YearMonthIntervalType(),
|
||||||
Period.of(0, 1, 0))
|
Period.of(0, 1, 0))
|
||||||
|
|
||||||
val intervalType: DataType = YearMonthIntervalType
|
val intervalType: DataType = YearMonthIntervalType()
|
||||||
|
|
||||||
def splitStep(input: Any): (Int, Int, Long) = {
|
def splitStep(input: Any): (Int, Int, Long) = {
|
||||||
(input.asInstanceOf[Int], 0, 0)
|
(input.asInstanceOf[Int], 0, 0)
|
||||||
|
|
|
@ -2396,7 +2396,7 @@ object DatePart {
|
||||||
throw QueryCompilationErrors.literalTypeUnsupportedForSourceTypeError(fieldStr, source)
|
throw QueryCompilationErrors.literalTypeUnsupportedForSourceTypeError(fieldStr, source)
|
||||||
|
|
||||||
source.dataType match {
|
source.dataType match {
|
||||||
case YearMonthIntervalType | _: DayTimeIntervalType | CalendarIntervalType =>
|
case _: YearMonthIntervalType | _: DayTimeIntervalType | CalendarIntervalType =>
|
||||||
ExtractIntervalPart.parseExtractField(fieldStr, source, analysisException)
|
ExtractIntervalPart.parseExtractField(fieldStr, source, analysisException)
|
||||||
case _ =>
|
case _ =>
|
||||||
DatePart.parseExtractField(fieldStr, source, analysisException)
|
DatePart.parseExtractField(fieldStr, source, analysisException)
|
||||||
|
|
|
@ -496,7 +496,7 @@ abstract class HashExpression[E] extends Expression {
|
||||||
case d: DecimalType => genHashDecimal(ctx, d, input, result)
|
case d: DecimalType => genHashDecimal(ctx, d, input, result)
|
||||||
case CalendarIntervalType => genHashCalendarInterval(input, result)
|
case CalendarIntervalType => genHashCalendarInterval(input, result)
|
||||||
case _: DayTimeIntervalType => genHashLong(input, result)
|
case _: DayTimeIntervalType => genHashLong(input, result)
|
||||||
case YearMonthIntervalType => genHashInt(input, result)
|
case _: YearMonthIntervalType => genHashInt(input, result)
|
||||||
case BinaryType => genHashBytes(input, result)
|
case BinaryType => genHashBytes(input, result)
|
||||||
case StringType => genHashString(input, result)
|
case StringType => genHashString(input, result)
|
||||||
case ArrayType(et, containsNull) => genHashForArray(ctx, input, result, et, containsNull)
|
case ArrayType(et, containsNull) => genHashForArray(ctx, input, result, et, containsNull)
|
||||||
|
|
|
@ -125,11 +125,11 @@ object ExtractIntervalPart {
|
||||||
source: Expression,
|
source: Expression,
|
||||||
errorHandleFunc: => Nothing): Expression = {
|
errorHandleFunc: => Nothing): Expression = {
|
||||||
(extractField.toUpperCase(Locale.ROOT), source.dataType) match {
|
(extractField.toUpperCase(Locale.ROOT), source.dataType) match {
|
||||||
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", YearMonthIntervalType) =>
|
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", _: YearMonthIntervalType) =>
|
||||||
ExtractANSIIntervalYears(source)
|
ExtractANSIIntervalYears(source)
|
||||||
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", CalendarIntervalType) =>
|
case ("YEAR" | "Y" | "YEARS" | "YR" | "YRS", CalendarIntervalType) =>
|
||||||
ExtractIntervalYears(source)
|
ExtractIntervalYears(source)
|
||||||
case ("MONTH" | "MON" | "MONS" | "MONTHS", YearMonthIntervalType) =>
|
case ("MONTH" | "MON" | "MONS" | "MONTHS", _: YearMonthIntervalType) =>
|
||||||
ExtractANSIIntervalMonths(source)
|
ExtractANSIIntervalMonths(source)
|
||||||
case ("MONTH" | "MON" | "MONS" | "MONTHS", CalendarIntervalType) =>
|
case ("MONTH" | "MON" | "MONS" | "MONTHS", CalendarIntervalType) =>
|
||||||
ExtractIntervalMonths(source)
|
ExtractIntervalMonths(source)
|
||||||
|
@ -374,7 +374,7 @@ case class MakeYMInterval(years: Expression, months: Expression)
|
||||||
override def left: Expression = years
|
override def left: Expression = years
|
||||||
override def right: Expression = months
|
override def right: Expression = months
|
||||||
override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType)
|
override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType)
|
||||||
override def dataType: DataType = YearMonthIntervalType
|
override def dataType: DataType = YearMonthIntervalType()
|
||||||
|
|
||||||
override def nullSafeEval(year: Any, month: Any): Any = {
|
override def nullSafeEval(year: Any, month: Any): Any = {
|
||||||
Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(),
|
Math.toIntExact(Math.addExact(month.asInstanceOf[Number].longValue(),
|
||||||
|
@ -407,7 +407,7 @@ case class MultiplyYMInterval(
|
||||||
override def right: Expression = num
|
override def right: Expression = num
|
||||||
|
|
||||||
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
|
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
|
||||||
override def dataType: DataType = YearMonthIntervalType
|
override def dataType: DataType = YearMonthIntervalType()
|
||||||
|
|
||||||
@transient
|
@transient
|
||||||
private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
|
private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
|
||||||
|
@ -517,7 +517,7 @@ case class DivideYMInterval(
|
||||||
override def right: Expression = num
|
override def right: Expression = num
|
||||||
|
|
||||||
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
|
override def inputTypes: Seq[AbstractDataType] = Seq(YearMonthIntervalType, NumericType)
|
||||||
override def dataType: DataType = YearMonthIntervalType
|
override def dataType: DataType = YearMonthIntervalType()
|
||||||
|
|
||||||
@transient
|
@transient
|
||||||
private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
|
private lazy val evalFunc: (Int, Any) => Any = right.dataType match {
|
||||||
|
|
|
@ -84,7 +84,7 @@ object Literal {
|
||||||
case ld: LocalDate => Literal(ld.toEpochDay.toInt, DateType)
|
case ld: LocalDate => Literal(ld.toEpochDay.toInt, DateType)
|
||||||
case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
|
case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
|
||||||
case d: Duration => Literal(durationToMicros(d), DayTimeIntervalType())
|
case d: Duration => Literal(durationToMicros(d), DayTimeIntervalType())
|
||||||
case p: Period => Literal(periodToMonths(p), YearMonthIntervalType)
|
case p: Period => Literal(periodToMonths(p), YearMonthIntervalType())
|
||||||
case a: Array[Byte] => Literal(a, BinaryType)
|
case a: Array[Byte] => Literal(a, BinaryType)
|
||||||
case a: collection.mutable.WrappedArray[_] => apply(a.array)
|
case a: collection.mutable.WrappedArray[_] => apply(a.array)
|
||||||
case a: Array[_] =>
|
case a: Array[_] =>
|
||||||
|
@ -122,7 +122,7 @@ object Literal {
|
||||||
case _ if clz == classOf[Timestamp] => TimestampType
|
case _ if clz == classOf[Timestamp] => TimestampType
|
||||||
case _ if clz == classOf[LocalDateTime] => TimestampWithoutTZType
|
case _ if clz == classOf[LocalDateTime] => TimestampWithoutTZType
|
||||||
case _ if clz == classOf[Duration] => DayTimeIntervalType()
|
case _ if clz == classOf[Duration] => DayTimeIntervalType()
|
||||||
case _ if clz == classOf[Period] => YearMonthIntervalType
|
case _ if clz == classOf[Period] => YearMonthIntervalType()
|
||||||
case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT
|
case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT
|
||||||
case _ if clz == classOf[Array[Byte]] => BinaryType
|
case _ if clz == classOf[Array[Byte]] => BinaryType
|
||||||
case _ if clz == classOf[Array[Char]] => StringType
|
case _ if clz == classOf[Array[Char]] => StringType
|
||||||
|
@ -181,7 +181,7 @@ object Literal {
|
||||||
case TimestampType => create(0L, TimestampType)
|
case TimestampType => create(0L, TimestampType)
|
||||||
case TimestampWithoutTZType => create(0L, TimestampWithoutTZType)
|
case TimestampWithoutTZType => create(0L, TimestampWithoutTZType)
|
||||||
case it: DayTimeIntervalType => create(0L, it)
|
case it: DayTimeIntervalType => create(0L, it)
|
||||||
case YearMonthIntervalType => create(0, YearMonthIntervalType)
|
case it: YearMonthIntervalType => create(0, it)
|
||||||
case StringType => Literal("")
|
case StringType => Literal("")
|
||||||
case BinaryType => Literal("".getBytes(StandardCharsets.UTF_8))
|
case BinaryType => Literal("".getBytes(StandardCharsets.UTF_8))
|
||||||
case CalendarIntervalType => Literal(new CalendarInterval(0, 0, 0))
|
case CalendarIntervalType => Literal(new CalendarInterval(0, 0, 0))
|
||||||
|
@ -200,7 +200,7 @@ object Literal {
|
||||||
case BooleanType => v.isInstanceOf[Boolean]
|
case BooleanType => v.isInstanceOf[Boolean]
|
||||||
case ByteType => v.isInstanceOf[Byte]
|
case ByteType => v.isInstanceOf[Byte]
|
||||||
case ShortType => v.isInstanceOf[Short]
|
case ShortType => v.isInstanceOf[Short]
|
||||||
case IntegerType | DateType | YearMonthIntervalType => v.isInstanceOf[Int]
|
case IntegerType | DateType | _: YearMonthIntervalType => v.isInstanceOf[Int]
|
||||||
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | TimestampWithoutTZType | _: DayTimeIntervalType =>
|
||||||
v.isInstanceOf[Long]
|
v.isInstanceOf[Long]
|
||||||
case FloatType => v.isInstanceOf[Float]
|
case FloatType => v.isInstanceOf[Float]
|
||||||
|
@ -348,7 +348,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
|
||||||
TimestampFormatter.getFractionFormatter(timeZoneId).format(value.asInstanceOf[Long])
|
TimestampFormatter.getFractionFormatter(timeZoneId).format(value.asInstanceOf[Long])
|
||||||
case DayTimeIntervalType(startField, endField) =>
|
case DayTimeIntervalType(startField, endField) =>
|
||||||
toDayTimeIntervalString(value.asInstanceOf[Long], ANSI_STYLE, startField, endField)
|
toDayTimeIntervalString(value.asInstanceOf[Long], ANSI_STYLE, startField, endField)
|
||||||
case YearMonthIntervalType => toYearMonthIntervalString(value.asInstanceOf[Int], ANSI_STYLE)
|
case YearMonthIntervalType(startField, endField) =>
|
||||||
|
toYearMonthIntervalString(value.asInstanceOf[Int], ANSI_STYLE, startField, endField)
|
||||||
case _ =>
|
case _ =>
|
||||||
other.toString
|
other.toString
|
||||||
}
|
}
|
||||||
|
@ -401,7 +402,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
|
||||||
ExprCode.forNonNullValue(JavaCode.literal(code, dataType))
|
ExprCode.forNonNullValue(JavaCode.literal(code, dataType))
|
||||||
}
|
}
|
||||||
dataType match {
|
dataType match {
|
||||||
case BooleanType | IntegerType | DateType | YearMonthIntervalType =>
|
case BooleanType | IntegerType | DateType | _: YearMonthIntervalType =>
|
||||||
toExprCode(value.toString)
|
toExprCode(value.toString)
|
||||||
case FloatType =>
|
case FloatType =>
|
||||||
value.asInstanceOf[Float] match {
|
value.asInstanceOf[Float] match {
|
||||||
|
@ -471,7 +472,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
|
||||||
case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'"
|
case (v: Array[Byte], BinaryType) => s"X'${DatatypeConverter.printHexBinary(v)}'"
|
||||||
case (i: Long, DayTimeIntervalType(startField, endField)) =>
|
case (i: Long, DayTimeIntervalType(startField, endField)) =>
|
||||||
toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)
|
toDayTimeIntervalString(i, ANSI_STYLE, startField, endField)
|
||||||
case (i: Int, YearMonthIntervalType) => toYearMonthIntervalString(i, ANSI_STYLE)
|
case (i: Int, YearMonthIntervalType(startField, endField)) =>
|
||||||
|
toYearMonthIntervalString(i, ANSI_STYLE, startField, endField)
|
||||||
case _ => value.toString
|
case _ => value.toString
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -101,9 +101,9 @@ case class WindowSpecDefinition(
|
||||||
|
|
||||||
private def isValidFrameType(ft: DataType): Boolean = (orderSpec.head.dataType, ft) match {
|
private def isValidFrameType(ft: DataType): Boolean = (orderSpec.head.dataType, ft) match {
|
||||||
case (DateType, IntegerType) => true
|
case (DateType, IntegerType) => true
|
||||||
case (DateType, YearMonthIntervalType) => true
|
case (DateType, _: YearMonthIntervalType) => true
|
||||||
case (TimestampType, CalendarIntervalType) => true
|
case (TimestampType, CalendarIntervalType) => true
|
||||||
case (TimestampType, YearMonthIntervalType) => true
|
case (TimestampType, _: YearMonthIntervalType) => true
|
||||||
case (TimestampType, _: DayTimeIntervalType) => true
|
case (TimestampType, _: DayTimeIntervalType) => true
|
||||||
case (a, b) => a == b
|
case (a, b) => a == b
|
||||||
}
|
}
|
||||||
|
|
|
@ -2354,7 +2354,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT)
|
val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT)
|
||||||
if (toUnit == "month") {
|
if (toUnit == "month") {
|
||||||
assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0)
|
assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0)
|
||||||
Literal(calendarInterval.months, YearMonthIntervalType)
|
// TODO(SPARK-35773): Parse year-month interval literals to tightest types
|
||||||
|
Literal(calendarInterval.months, YearMonthIntervalType())
|
||||||
} else {
|
} else {
|
||||||
assert(calendarInterval.months == 0)
|
assert(calendarInterval.months == 0)
|
||||||
val fromUnit =
|
val fromUnit =
|
||||||
|
@ -2513,7 +2514,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
}
|
}
|
||||||
|
|
||||||
override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = {
|
override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = {
|
||||||
YearMonthIntervalType
|
// TODO(SPARK-35774): Parse any year-month interval types in SQL
|
||||||
|
YearMonthIntervalType()
|
||||||
}
|
}
|
||||||
|
|
||||||
override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = {
|
override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = {
|
||||||
|
|
|
@ -106,7 +106,11 @@ object IntervalUtils {
|
||||||
private val yearMonthLiteralRegex =
|
private val yearMonthLiteralRegex =
|
||||||
(s"(?i)^INTERVAL\\s+([+|-])?'$yearMonthPatternString'\\s+YEAR\\s+TO\\s+MONTH$$").r
|
(s"(?i)^INTERVAL\\s+([+|-])?'$yearMonthPatternString'\\s+YEAR\\s+TO\\s+MONTH$$").r
|
||||||
|
|
||||||
def castStringToYMInterval(input: UTF8String): Int = {
|
def castStringToYMInterval(
|
||||||
|
input: UTF8String,
|
||||||
|
// TODO(SPARK-35768): Take into account year-month interval fields in cast
|
||||||
|
startField: Byte,
|
||||||
|
endField: Byte): Int = {
|
||||||
input.trimAll().toString match {
|
input.trimAll().toString match {
|
||||||
case yearMonthRegex("-", year, month) => toYMInterval(year, month, -1)
|
case yearMonthRegex("-", year, month) => toYMInterval(year, month, -1)
|
||||||
case yearMonthRegex(_, year, month) => toYMInterval(year, month, 1)
|
case yearMonthRegex(_, year, month) => toYMInterval(year, month, 1)
|
||||||
|
@ -934,9 +938,16 @@ object IntervalUtils {
|
||||||
*
|
*
|
||||||
* @param months The number of months, positive or negative
|
* @param months The number of months, positive or negative
|
||||||
* @param style The style of textual representation of the interval
|
* @param style The style of textual representation of the interval
|
||||||
|
* @param startField The start field (YEAR or MONTH) which the interval comprises of.
|
||||||
|
* @param endField The end field (YEAR or MONTH) which the interval comprises of.
|
||||||
* @return Year-month interval string
|
* @return Year-month interval string
|
||||||
*/
|
*/
|
||||||
def toYearMonthIntervalString(months: Int, style: IntervalStyle): String = {
|
def toYearMonthIntervalString(
|
||||||
|
months: Int,
|
||||||
|
style: IntervalStyle,
|
||||||
|
// TODO(SPARK-35771): Format year-month intervals using type fields
|
||||||
|
startField: Byte,
|
||||||
|
endField: Byte): String = {
|
||||||
var sign = ""
|
var sign = ""
|
||||||
var absMonths: Long = months
|
var absMonths: Long = months
|
||||||
if (months < 0) {
|
if (months < 0) {
|
||||||
|
@ -956,6 +967,8 @@ object IntervalUtils {
|
||||||
*
|
*
|
||||||
* @param micros The number of microseconds, positive or negative
|
* @param micros The number of microseconds, positive or negative
|
||||||
* @param style The style of textual representation of the interval
|
* @param style The style of textual representation of the interval
|
||||||
|
* @param startField The start field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
|
||||||
|
* @param endField The end field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
|
||||||
* @return Day-time interval string
|
* @return Day-time interval string
|
||||||
*/
|
*/
|
||||||
def toDayTimeIntervalString(
|
def toDayTimeIntervalString(
|
||||||
|
|
|
@ -63,7 +63,7 @@ object TypeUtils {
|
||||||
|
|
||||||
def checkForAnsiIntervalOrNumericType(
|
def checkForAnsiIntervalOrNumericType(
|
||||||
dt: DataType, funcName: String): TypeCheckResult = dt match {
|
dt: DataType, funcName: String): TypeCheckResult = dt match {
|
||||||
case YearMonthIntervalType | _: DayTimeIntervalType | NullType =>
|
case _: YearMonthIntervalType | _: DayTimeIntervalType | NullType =>
|
||||||
TypeCheckResult.TypeCheckSuccess
|
TypeCheckResult.TypeCheckSuccess
|
||||||
case dt if dt.isInstanceOf[NumericType] => TypeCheckResult.TypeCheckSuccess
|
case dt if dt.isInstanceOf[NumericType] => TypeCheckResult.TypeCheckSuccess
|
||||||
case other => TypeCheckResult.TypeCheckFailure(
|
case other => TypeCheckResult.TypeCheckFailure(
|
||||||
|
@ -117,7 +117,7 @@ object TypeUtils {
|
||||||
|
|
||||||
def invokeOnceForInterval(dataType: DataType)(f: => Unit): Unit = {
|
def invokeOnceForInterval(dataType: DataType)(f: => Unit): Unit = {
|
||||||
def isInterval(dataType: DataType): Boolean = dataType match {
|
def isInterval(dataType: DataType): Boolean = dataType match {
|
||||||
case CalendarIntervalType | _: DayTimeIntervalType | YearMonthIntervalType => true
|
case CalendarIntervalType | _: DayTimeIntervalType | _: YearMonthIntervalType => true
|
||||||
case _ => false
|
case _ => false
|
||||||
}
|
}
|
||||||
if (dataType.existsRecursively(isInterval)) f
|
if (dataType.existsRecursively(isInterval)) f
|
||||||
|
|
|
@ -1636,4 +1636,15 @@ private[spark] object QueryCompilationErrors {
|
||||||
def invalidDayTimeIntervalType(startFieldName: String, endFieldName: String): Throwable = {
|
def invalidDayTimeIntervalType(startFieldName: String, endFieldName: String): Throwable = {
|
||||||
new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.")
|
new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def invalidYearMonthField(field: Byte): Throwable = {
|
||||||
|
val supportedIds = YearMonthIntervalType.yearMonthFields
|
||||||
|
.map(i => s"$i (${YearMonthIntervalType.fieldToString(i)})")
|
||||||
|
new AnalysisException(s"Invalid field id '$field' in year-month interval. " +
|
||||||
|
s"Supported interval fields: ${supportedIds.mkString(", ")}.")
|
||||||
|
}
|
||||||
|
|
||||||
|
def invalidYearMonthIntervalType(startFieldName: String, endFieldName: String): Throwable = {
|
||||||
|
new AnalysisException(s"'interval $startFieldName to $endFieldName' is invalid.")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -172,7 +172,9 @@ object DataType {
|
||||||
Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
|
Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
|
||||||
DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
|
DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
|
||||||
// TODO(SPARK-35732): Parse DayTimeIntervalType from JSON
|
// TODO(SPARK-35732): Parse DayTimeIntervalType from JSON
|
||||||
DayTimeIntervalType(), YearMonthIntervalType, TimestampWithoutTZType)
|
DayTimeIntervalType(),
|
||||||
|
// TODO(SPARK-35770): Parse YearMonthIntervalType from JSON
|
||||||
|
YearMonthIntervalType(), TimestampWithoutTZType)
|
||||||
.map(t => t.typeName -> t).toMap
|
.map(t => t.typeName -> t).toMap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,8 @@ import scala.math.Ordering
|
||||||
import scala.reflect.runtime.universe.typeTag
|
import scala.reflect.runtime.universe.typeTag
|
||||||
|
|
||||||
import org.apache.spark.annotation.Unstable
|
import org.apache.spark.annotation.Unstable
|
||||||
|
import org.apache.spark.sql.errors.QueryCompilationErrors
|
||||||
|
import org.apache.spark.sql.types.YearMonthIntervalType.fieldToString
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The type represents year-month intervals of the SQL standard. A year-month interval is made up
|
* The type represents year-month intervals of the SQL standard. A year-month interval is made up
|
||||||
|
@ -30,12 +32,15 @@ import org.apache.spark.annotation.Unstable
|
||||||
*
|
*
|
||||||
* `YearMonthIntervalType` represents positive as well as negative year-month intervals.
|
* `YearMonthIntervalType` represents positive as well as negative year-month intervals.
|
||||||
*
|
*
|
||||||
* Please use the singleton `DataTypes.YearMonthIntervalType` to refer the type.
|
* @param startField The leftmost field which the type comprises of. Valid values:
|
||||||
|
* 0 (YEAR), 1 (MONTH).
|
||||||
|
* @param endField The rightmost field which the type comprises of. Valid values:
|
||||||
|
* 0 (YEAR), 1 (MONTH).
|
||||||
*
|
*
|
||||||
* @since 3.2.0
|
* @since 3.2.0
|
||||||
*/
|
*/
|
||||||
@Unstable
|
@Unstable
|
||||||
class YearMonthIntervalType private() extends AtomicType {
|
case class YearMonthIntervalType(startField: Byte, endField: Byte) extends AtomicType {
|
||||||
/**
|
/**
|
||||||
* Internally, values of year-month intervals are stored in `Int` values as amount of months
|
* Internally, values of year-month intervals are stored in `Int` values as amount of months
|
||||||
* that are calculated by the formula:
|
* that are calculated by the formula:
|
||||||
|
@ -55,16 +60,47 @@ class YearMonthIntervalType private() extends AtomicType {
|
||||||
|
|
||||||
private[spark] override def asNullable: YearMonthIntervalType = this
|
private[spark] override def asNullable: YearMonthIntervalType = this
|
||||||
|
|
||||||
override def typeName: String = "interval year to month"
|
override val typeName: String = {
|
||||||
|
val startFieldName = fieldToString(startField)
|
||||||
|
val endFieldName = fieldToString(endField)
|
||||||
|
if (startFieldName == endFieldName) {
|
||||||
|
s"interval $startFieldName"
|
||||||
|
} else if (startField < endField) {
|
||||||
|
s"interval $startFieldName to $endFieldName"
|
||||||
|
} else {
|
||||||
|
throw QueryCompilationErrors.invalidDayTimeIntervalType(startFieldName, endFieldName)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The companion case object and its class is separated so the companion object also subclasses
|
* Extra factory methods and pattern matchers for YearMonthIntervalType.
|
||||||
* the YearMonthIntervalType class. Otherwise, the companion object would be of type
|
|
||||||
* "YearMonthIntervalType$" in byte code. Defined with a private constructor so the companion object
|
|
||||||
* is the only possible instantiation.
|
|
||||||
*
|
*
|
||||||
* @since 3.2.0
|
* @since 3.2.0
|
||||||
*/
|
*/
|
||||||
@Unstable
|
@Unstable
|
||||||
case object YearMonthIntervalType extends YearMonthIntervalType
|
case object YearMonthIntervalType extends AbstractDataType {
|
||||||
|
val YEAR: Byte = 0
|
||||||
|
val MONTH: Byte = 1
|
||||||
|
val yearMonthFields = Seq(YEAR, MONTH)
|
||||||
|
|
||||||
|
def fieldToString(field: Byte): String = field match {
|
||||||
|
case YEAR => "year"
|
||||||
|
case MONTH => "month"
|
||||||
|
case invalid => throw QueryCompilationErrors.invalidYearMonthField(invalid)
|
||||||
|
}
|
||||||
|
|
||||||
|
val stringToField: Map[String, Byte] = yearMonthFields.map(i => fieldToString(i) -> i).toMap
|
||||||
|
|
||||||
|
val DEFAULT = YearMonthIntervalType(YEAR, MONTH)
|
||||||
|
|
||||||
|
def apply(): YearMonthIntervalType = DEFAULT
|
||||||
|
|
||||||
|
override private[sql] def defaultConcreteType: DataType = DEFAULT
|
||||||
|
|
||||||
|
override private[sql] def acceptsType(other: DataType): Boolean = {
|
||||||
|
other.isInstanceOf[YearMonthIntervalType]
|
||||||
|
}
|
||||||
|
|
||||||
|
override private[sql] def simpleString: String = defaultConcreteType.simpleString
|
||||||
|
}
|
||||||
|
|
|
@ -54,7 +54,7 @@ private[sql] object ArrowUtils {
|
||||||
new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
|
new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
|
||||||
}
|
}
|
||||||
case NullType => ArrowType.Null.INSTANCE
|
case NullType => ArrowType.Null.INSTANCE
|
||||||
case YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
|
case _: YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
|
||||||
case _: DayTimeIntervalType => new ArrowType.Interval(IntervalUnit.DAY_TIME)
|
case _: DayTimeIntervalType => new ArrowType.Interval(IntervalUnit.DAY_TIME)
|
||||||
case _ =>
|
case _ =>
|
||||||
throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
|
throw new UnsupportedOperationException(s"Unsupported data type: ${dt.catalogString}")
|
||||||
|
@ -76,7 +76,7 @@ private[sql] object ArrowUtils {
|
||||||
case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType
|
case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType
|
||||||
case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType
|
case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType
|
||||||
case ArrowType.Null.INSTANCE => NullType
|
case ArrowType.Null.INSTANCE => NullType
|
||||||
case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType
|
case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType()
|
||||||
case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType()
|
case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType()
|
||||||
case _ => throw new UnsupportedOperationException(s"Unsupported data type: $dt")
|
case _ => throw new UnsupportedOperationException(s"Unsupported data type: $dt")
|
||||||
}
|
}
|
||||||
|
|
|
@ -284,7 +284,7 @@ object RandomDataGenerator {
|
||||||
new CalendarInterval(months, days, ns)
|
new CalendarInterval(months, days, ns)
|
||||||
})
|
})
|
||||||
case _: DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS))
|
case _: DayTimeIntervalType => Some(() => Duration.of(rand.nextLong(), ChronoUnit.MICROS))
|
||||||
case YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized())
|
case _: YearMonthIntervalType => Some(() => Period.ofMonths(rand.nextInt()).normalized())
|
||||||
case DecimalType.Fixed(precision, scale) => Some(
|
case DecimalType.Fixed(precision, scale) => Some(
|
||||||
() => BigDecimal.apply(
|
() => BigDecimal.apply(
|
||||||
rand.nextLong() % math.pow(10, precision).toLong,
|
rand.nextLong() % math.pow(10, precision).toLong,
|
||||||
|
|
|
@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.CatalystTypeConverters
|
||||||
import org.apache.spark.sql.catalyst.plans.SQLHelper
|
import org.apache.spark.sql.catalyst.plans.SQLHelper
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests of [[RandomDataGenerator]].
|
* Tests of [[RandomDataGenerator]].
|
||||||
|
@ -146,7 +146,7 @@ class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") {
|
test("SPARK-35116: The generated data fits the precision of DayTimeIntervalType in spark") {
|
||||||
(dayTimeIntervalTypes :+ YearMonthIntervalType).foreach { dt =>
|
(dayTimeIntervalTypes ++ yearMonthIntervalTypes).foreach { dt =>
|
||||||
for (seed <- 1 to 1000) {
|
for (seed <- 1 to 1000) {
|
||||||
val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get
|
val generator = RandomDataGenerator.forType(dt, false, new Random(seed)).get
|
||||||
val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt)
|
val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dt)
|
||||||
|
|
|
@ -320,7 +320,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
|
||||||
val months = sign * input
|
val months = sign * input
|
||||||
val period = IntervalUtils.monthsToPeriod(months)
|
val period = IntervalUtils.monthsToPeriod(months)
|
||||||
assert(
|
assert(
|
||||||
CatalystTypeConverters.createToScalaConverter(YearMonthIntervalType)(months) === period)
|
CatalystTypeConverters.createToScalaConverter(YearMonthIntervalType())(months) === period)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
|
||||||
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils}
|
import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData, IntervalUtils}
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
|
|
||||||
@SQLUserDefinedType(udt = classOf[ExamplePointUDT])
|
@SQLUserDefinedType(udt = classOf[ExamplePointUDT])
|
||||||
class ExamplePoint(val x: Double, val y: Double) extends Serializable {
|
class ExamplePoint(val x: Double, val y: Double) extends Serializable {
|
||||||
|
@ -366,13 +366,15 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-34615: encoding/decoding YearMonthIntervalType to/from java.time.Period") {
|
test("SPARK-34615: encoding/decoding YearMonthIntervalType to/from java.time.Period") {
|
||||||
val schema = new StructType().add("p", YearMonthIntervalType)
|
yearMonthIntervalTypes.foreach { yearMonthIntervalType =>
|
||||||
val encoder = RowEncoder(schema).resolveAndBind()
|
val schema = new StructType().add("p", yearMonthIntervalType)
|
||||||
val period = java.time.Period.ofMonths(1)
|
val encoder = RowEncoder(schema).resolveAndBind()
|
||||||
val row = toRow(encoder, Row(period))
|
val period = java.time.Period.ofMonths(1)
|
||||||
assert(row.getInt(0) === IntervalUtils.periodToMonths(period))
|
val row = toRow(encoder, Row(period))
|
||||||
val readback = fromRow(encoder, row)
|
assert(row.getInt(0) === IntervalUtils.periodToMonths(period))
|
||||||
assert(readback.get(0).equals(period))
|
val readback = fromRow(encoder, row)
|
||||||
|
assert(readback.get(0).equals(period))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
|
|
@ -614,20 +614,20 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
|
||||||
Seq(true, false).foreach { failOnError =>
|
Seq(true, false).foreach { failOnError =>
|
||||||
checkExceptionInExpression[ArithmeticException](
|
checkExceptionInExpression[ArithmeticException](
|
||||||
UnaryMinus(
|
UnaryMinus(
|
||||||
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType),
|
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType()),
|
||||||
failOnError),
|
failOnError),
|
||||||
"overflow")
|
"overflow")
|
||||||
checkExceptionInExpression[ArithmeticException](
|
checkExceptionInExpression[ArithmeticException](
|
||||||
Subtract(
|
Subtract(
|
||||||
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType),
|
Literal.create(Period.ofMonths(Int.MinValue), YearMonthIntervalType()),
|
||||||
Literal.create(Period.ofMonths(10), YearMonthIntervalType),
|
Literal.create(Period.ofMonths(10), YearMonthIntervalType()),
|
||||||
failOnError
|
failOnError
|
||||||
),
|
),
|
||||||
"overflow")
|
"overflow")
|
||||||
checkExceptionInExpression[ArithmeticException](
|
checkExceptionInExpression[ArithmeticException](
|
||||||
Add(
|
Add(
|
||||||
Literal.create(Period.ofMonths(Int.MaxValue), YearMonthIntervalType),
|
Literal.create(Period.ofMonths(Int.MaxValue), YearMonthIntervalType()),
|
||||||
Literal.create(Period.ofMonths(10), YearMonthIntervalType),
|
Literal.create(Period.ofMonths(10), YearMonthIntervalType()),
|
||||||
failOnError
|
failOnError
|
||||||
),
|
),
|
||||||
"overflow")
|
"overflow")
|
||||||
|
|
|
@ -612,43 +612,43 @@ class CastSuite extends CastSuiteBase {
|
||||||
|
|
||||||
test("SPARK-35111: Cast string to year-month interval") {
|
test("SPARK-35111: Cast string to year-month interval") {
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL '1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL '1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), 12)
|
YearMonthIntervalType()), 12)
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL '-1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL '-1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), -12)
|
YearMonthIntervalType()), -12)
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL -'-1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL -'-1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), 12)
|
YearMonthIntervalType()), 12)
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL +'-1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL +'-1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), -12)
|
YearMonthIntervalType()), -12)
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL +'+1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL +'+1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), 12)
|
YearMonthIntervalType()), 12)
|
||||||
checkEvaluation(cast(Literal.create("INTERVAL +'1-0' YEAR TO MONTH"),
|
checkEvaluation(cast(Literal.create("INTERVAL +'1-0' YEAR TO MONTH"),
|
||||||
YearMonthIntervalType), 12)
|
YearMonthIntervalType()), 12)
|
||||||
checkEvaluation(cast(Literal.create(" interval +'1-0' YEAR TO MONTH "),
|
checkEvaluation(cast(Literal.create(" interval +'1-0' YEAR TO MONTH "),
|
||||||
YearMonthIntervalType), 12)
|
YearMonthIntervalType()), 12)
|
||||||
checkEvaluation(cast(Literal.create(" -1-0 "), YearMonthIntervalType), -12)
|
checkEvaluation(cast(Literal.create(" -1-0 "), YearMonthIntervalType()), -12)
|
||||||
checkEvaluation(cast(Literal.create("-1-0"), YearMonthIntervalType), -12)
|
checkEvaluation(cast(Literal.create("-1-0"), YearMonthIntervalType()), -12)
|
||||||
checkEvaluation(cast(Literal.create(null, StringType), YearMonthIntervalType), null)
|
checkEvaluation(cast(Literal.create(null, StringType), YearMonthIntervalType()), null)
|
||||||
|
|
||||||
Seq("0-0", "10-1", "-178956970-7", "178956970-7", "-178956970-8").foreach { interval =>
|
Seq("0-0", "10-1", "-178956970-7", "178956970-7", "-178956970-8").foreach { interval =>
|
||||||
val ansiInterval = s"INTERVAL '$interval' YEAR TO MONTH"
|
val ansiInterval = s"INTERVAL '$interval' YEAR TO MONTH"
|
||||||
checkEvaluation(
|
checkEvaluation(
|
||||||
cast(cast(Literal.create(interval), YearMonthIntervalType), StringType), ansiInterval)
|
cast(cast(Literal.create(interval), YearMonthIntervalType()), StringType), ansiInterval)
|
||||||
checkEvaluation(cast(cast(Literal.create(ansiInterval),
|
checkEvaluation(cast(cast(Literal.create(ansiInterval),
|
||||||
YearMonthIntervalType), StringType), ansiInterval)
|
YearMonthIntervalType()), StringType), ansiInterval)
|
||||||
}
|
}
|
||||||
|
|
||||||
Seq("INTERVAL '-178956970-9' YEAR TO MONTH", "INTERVAL '178956970-8' YEAR TO MONTH")
|
Seq("INTERVAL '-178956970-9' YEAR TO MONTH", "INTERVAL '178956970-8' YEAR TO MONTH")
|
||||||
.foreach { interval =>
|
.foreach { interval =>
|
||||||
val e = intercept[IllegalArgumentException] {
|
val e = intercept[IllegalArgumentException] {
|
||||||
cast(Literal.create(interval), YearMonthIntervalType).eval()
|
cast(Literal.create(interval), YearMonthIntervalType()).eval()
|
||||||
}.getMessage
|
}.getMessage
|
||||||
assert(e.contains("Error parsing interval year-month string: integer overflow"))
|
assert(e.contains("Error parsing interval year-month string: integer overflow"))
|
||||||
}
|
}
|
||||||
|
|
||||||
Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue)
|
Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue)
|
||||||
.foreach { period =>
|
.foreach { period =>
|
||||||
val interval = Literal.create(Period.ofMonths(period), YearMonthIntervalType)
|
val interval = Literal.create(Period.ofMonths(period), YearMonthIntervalType())
|
||||||
checkEvaluation(cast(cast(interval, StringType), YearMonthIntervalType), period)
|
checkEvaluation(cast(cast(interval, StringType), YearMonthIntervalType()), period)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils._
|
||||||
import org.apache.spark.sql.catalyst.util.IntervalUtils.microsToDuration
|
import org.apache.spark.sql.catalyst.util.IntervalUtils.microsToDuration
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
import org.apache.spark.unsafe.types.UTF8String
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
||||||
abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
|
abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
@ -831,8 +831,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
s"INTERVAL '$intervalPayload' YEAR TO MONTH")
|
s"INTERVAL '$intervalPayload' YEAR TO MONTH")
|
||||||
}
|
}
|
||||||
|
|
||||||
checkConsistencyBetweenInterpretedAndCodegen(
|
yearMonthIntervalTypes.foreach { it =>
|
||||||
(child: Expression) => Cast(child, StringType), YearMonthIntervalType)
|
checkConsistencyBetweenInterpretedAndCodegen(
|
||||||
|
(child: Expression) => Cast(child, StringType), it)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-34668: cast day-time interval to string") {
|
test("SPARK-34668: cast day-time interval to string") {
|
||||||
|
|
|
@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
|
||||||
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
|
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, TimeZoneUTC}
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
||||||
|
|
||||||
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
@ -526,7 +526,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
private def testAddMonths(dataType: DataType): Unit = {
|
private def testAddMonths(dataType: DataType): Unit = {
|
||||||
def addMonths(date: Literal, months: Any): AddMonthsBase = dataType match {
|
def addMonths(date: Literal, months: Any): AddMonthsBase = dataType match {
|
||||||
case IntegerType => AddMonths(date, Literal.create(months, dataType))
|
case IntegerType => AddMonths(date, Literal.create(months, dataType))
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
val period = if (months == null) null else Period.ofMonths(months.asInstanceOf[Int])
|
val period = if (months == null) null else Period.ofMonths(months.asInstanceOf[Int])
|
||||||
DateAddYMInterval(date, Literal.create(period, dataType))
|
DateAddYMInterval(date, Literal.create(period, dataType))
|
||||||
}
|
}
|
||||||
|
@ -561,7 +561,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-34721: add a year-month interval to a date") {
|
test("SPARK-34721: add a year-month interval to a date") {
|
||||||
testAddMonths(YearMonthIntervalType)
|
testAddMonths(YearMonthIntervalType())
|
||||||
// Test evaluation results between Interpreted mode and Codegen mode
|
// Test evaluation results between Interpreted mode and Codegen mode
|
||||||
forAll (
|
forAll (
|
||||||
LiteralGenerator.randomGen(DateType),
|
LiteralGenerator.randomGen(DateType),
|
||||||
|
@ -1596,18 +1596,21 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(
|
checkEvaluation(
|
||||||
TimestampAddYMInterval(
|
TimestampAddYMInterval(
|
||||||
Literal(new Timestamp(sdf.parse("2016-01-29 10:00:00.000").getTime)),
|
Literal(new Timestamp(sdf.parse("2016-01-29 10:00:00.000").getTime)),
|
||||||
Literal.create(null, YearMonthIntervalType),
|
Literal.create(null, YearMonthIntervalType()),
|
||||||
timeZoneId),
|
timeZoneId),
|
||||||
null)
|
null)
|
||||||
checkEvaluation(
|
checkEvaluation(
|
||||||
TimestampAddYMInterval(
|
TimestampAddYMInterval(
|
||||||
Literal.create(null, TimestampType),
|
Literal.create(null, TimestampType),
|
||||||
Literal.create(null, YearMonthIntervalType),
|
Literal.create(null, YearMonthIntervalType()),
|
||||||
timeZoneId),
|
timeZoneId),
|
||||||
null)
|
null)
|
||||||
checkConsistencyBetweenInterpretedAndCodegen(
|
yearMonthIntervalTypes.foreach { it =>
|
||||||
(ts: Expression, interval: Expression) => TimestampAddYMInterval(ts, interval, timeZoneId),
|
checkConsistencyBetweenInterpretedAndCodegen(
|
||||||
TimestampType, YearMonthIntervalType)
|
(ts: Expression, interval: Expression) =>
|
||||||
|
TimestampAddYMInterval(ts, interval, timeZoneId),
|
||||||
|
TimestampType, it)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -699,7 +699,7 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
|
||||||
test("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") {
|
test("SPARK-35113: HashExpression support DayTimeIntervalType/YearMonthIntervalType") {
|
||||||
val dayTime = Literal.create(Duration.ofSeconds(1237123123), DayTimeIntervalType())
|
val dayTime = Literal.create(Duration.ofSeconds(1237123123), DayTimeIntervalType())
|
||||||
val yearMonth = Literal.create(Period.ofMonths(1234), YearMonthIntervalType)
|
val yearMonth = Literal.create(Period.ofMonths(1234), YearMonthIntervalType())
|
||||||
checkEvaluation(Murmur3Hash(Seq(dayTime), 10), -428664612)
|
checkEvaluation(Murmur3Hash(Seq(dayTime), 10), -428664612)
|
||||||
checkEvaluation(Murmur3Hash(Seq(yearMonth), 10), -686520021)
|
checkEvaluation(Murmur3Hash(Seq(yearMonth), 10), -686520021)
|
||||||
checkEvaluation(XxHash64(Seq(dayTime), 10), 8228802290839366895L)
|
checkEvaluation(XxHash64(Seq(dayTime), 10), 8228802290839366895L)
|
||||||
|
|
|
@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
|
||||||
import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
|
import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType}
|
import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, YearMonthIntervalType}
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes}
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, numericTypes, yearMonthIntervalTypes}
|
||||||
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
|
||||||
|
|
||||||
class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
@ -280,6 +280,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
|
||||||
test("SPARK-34824: multiply year-month interval by numeric") {
|
test("SPARK-34824: multiply year-month interval by numeric") {
|
||||||
Seq(
|
Seq(
|
||||||
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
|
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
|
||||||
|
@ -307,9 +308,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
numericTypes.foreach { numType =>
|
numericTypes.foreach { numType =>
|
||||||
checkConsistencyBetweenInterpretedAndCodegenAllowingException(
|
yearMonthIntervalTypes.foreach { it =>
|
||||||
(interval: Expression, num: Expression) => MultiplyYMInterval(interval, num),
|
checkConsistencyBetweenInterpretedAndCodegenAllowingException(
|
||||||
YearMonthIntervalType, numType)
|
(interval: Expression, num: Expression) => MultiplyYMInterval(interval, num),
|
||||||
|
it, numType)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -349,6 +352,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO(SPARK-35778): Check multiply/divide of year-month intervals of any fields by numeric
|
||||||
test("SPARK-34868: divide year-month interval by numeric") {
|
test("SPARK-34868: divide year-month interval by numeric") {
|
||||||
Seq(
|
Seq(
|
||||||
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
|
(Period.ofYears(-123), Literal(null, DecimalType.USER_DEFAULT)) -> null,
|
||||||
|
@ -376,9 +380,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
numericTypes.foreach { numType =>
|
numericTypes.foreach { numType =>
|
||||||
checkConsistencyBetweenInterpretedAndCodegenAllowingException(
|
yearMonthIntervalTypes.foreach { it =>
|
||||||
(interval: Expression, num: Expression) => DivideYMInterval(interval, num),
|
checkConsistencyBetweenInterpretedAndCodegenAllowingException(
|
||||||
YearMonthIntervalType, numType)
|
(interval: Expression, num: Expression) => DivideYMInterval(interval, num),
|
||||||
|
it, numType)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -429,8 +435,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(ExtractANSIIntervalMonths(Literal(p)),
|
checkEvaluation(ExtractANSIIntervalMonths(Literal(p)),
|
||||||
IntervalUtils.getMonths(p.toTotalMonths.toInt))
|
IntervalUtils.getMonths(p.toTotalMonths.toInt))
|
||||||
}
|
}
|
||||||
checkEvaluation(ExtractANSIIntervalYears(Literal(null, YearMonthIntervalType)), null)
|
checkEvaluation(ExtractANSIIntervalYears(Literal(null, YearMonthIntervalType())), null)
|
||||||
checkEvaluation(ExtractANSIIntervalMonths(Literal(null, YearMonthIntervalType)), null)
|
checkEvaluation(ExtractANSIIntervalMonths(Literal(null, YearMonthIntervalType())), null)
|
||||||
}
|
}
|
||||||
|
|
||||||
test("ANSI: extract days, hours, minutes and seconds") {
|
test("ANSI: extract days, hours, minutes and seconds") {
|
||||||
|
|
|
@ -50,7 +50,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(Literal.create(null, DateType), null)
|
checkEvaluation(Literal.create(null, DateType), null)
|
||||||
checkEvaluation(Literal.create(null, TimestampType), null)
|
checkEvaluation(Literal.create(null, TimestampType), null)
|
||||||
checkEvaluation(Literal.create(null, CalendarIntervalType), null)
|
checkEvaluation(Literal.create(null, CalendarIntervalType), null)
|
||||||
checkEvaluation(Literal.create(null, YearMonthIntervalType), null)
|
checkEvaluation(Literal.create(null, YearMonthIntervalType()), null)
|
||||||
checkEvaluation(Literal.create(null, DayTimeIntervalType()), null)
|
checkEvaluation(Literal.create(null, DayTimeIntervalType()), null)
|
||||||
checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null)
|
checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null)
|
||||||
checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null)
|
checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null)
|
||||||
|
@ -79,7 +79,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
|
checkEvaluation(Literal.default(TimestampType), Instant.ofEpochSecond(0))
|
||||||
}
|
}
|
||||||
checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L))
|
checkEvaluation(Literal.default(CalendarIntervalType), new CalendarInterval(0, 0, 0L))
|
||||||
checkEvaluation(Literal.default(YearMonthIntervalType), 0)
|
checkEvaluation(Literal.default(YearMonthIntervalType()), 0)
|
||||||
checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
|
checkEvaluation(Literal.default(DayTimeIntervalType()), 0L)
|
||||||
checkEvaluation(Literal.default(ArrayType(StringType)), Array())
|
checkEvaluation(Literal.default(ArrayType(StringType)), Array())
|
||||||
checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
|
checkEvaluation(Literal.default(MapType(IntegerType, StringType)), Map())
|
||||||
|
@ -345,7 +345,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
assert(Literal(Array("1", "2", "3")) ==
|
assert(Literal(Array("1", "2", "3")) ==
|
||||||
Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
|
Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
|
||||||
assert(Literal(Array(Period.ofMonths(1))) ==
|
assert(Literal(Array(Period.ofMonths(1))) ==
|
||||||
Literal.create(Array(Period.ofMonths(1)), ArrayType(YearMonthIntervalType)))
|
Literal.create(Array(Period.ofMonths(1)), ArrayType(YearMonthIntervalType())))
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-34342: Date/Timestamp toString") {
|
test("SPARK-34342: Date/Timestamp toString") {
|
||||||
|
|
|
@ -184,7 +184,7 @@ object LiteralGenerator {
|
||||||
|
|
||||||
lazy val yearMonthIntervalLiteralGen: Gen[Literal] = {
|
lazy val yearMonthIntervalLiteralGen: Gen[Literal] = {
|
||||||
for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) }
|
for { months <- Gen.choose(-1 * maxIntervalInMonths, maxIntervalInMonths) }
|
||||||
yield Literal.create(Period.ofMonths(months), YearMonthIntervalType)
|
yield Literal.create(Period.ofMonths(months), YearMonthIntervalType())
|
||||||
}
|
}
|
||||||
|
|
||||||
def randomGen(dt: DataType): Gen[Literal] = {
|
def randomGen(dt: DataType): Gen[Literal] = {
|
||||||
|
@ -204,7 +204,7 @@ object LiteralGenerator {
|
||||||
case CalendarIntervalType => calendarIntervalLiterGen
|
case CalendarIntervalType => calendarIntervalLiterGen
|
||||||
case DecimalType.Fixed(precision, scale) => decimalLiteralGen(precision, scale)
|
case DecimalType.Fixed(precision, scale) => decimalLiteralGen(precision, scale)
|
||||||
case _: DayTimeIntervalType => dayTimeIntervalLiteralGen
|
case _: DayTimeIntervalType => dayTimeIntervalLiteralGen
|
||||||
case YearMonthIntervalType => yearMonthIntervalLiteralGen
|
case _: YearMonthIntervalType => yearMonthIntervalLiteralGen
|
||||||
case dt => throw new IllegalArgumentException(s"not supported type $dt")
|
case dt => throw new IllegalArgumentException(s"not supported type $dt")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
|
||||||
import org.apache.spark.sql.catalyst.util.IntervalUtils
|
import org.apache.spark.sql.catalyst.util.IntervalUtils
|
||||||
import org.apache.spark.sql.internal.SQLConf
|
import org.apache.spark.sql.internal.SQLConf
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
import org.apache.spark.unsafe.Platform
|
import org.apache.spark.unsafe.Platform
|
||||||
import org.apache.spark.unsafe.types.UTF8String
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
||||||
|
@ -31,7 +31,7 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
|
||||||
val fixedLengthTypes = Array[DataType](
|
val fixedLengthTypes = Array[DataType](
|
||||||
BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
|
BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
|
||||||
DateType, TimestampType, YearMonthIntervalType) ++ dayTimeIntervalTypes
|
DateType, TimestampType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
|
||||||
|
|
||||||
val variableLengthTypes = Array(
|
val variableLengthTypes = Array(
|
||||||
StringType, DecimalType.defaultConcreteType, CalendarIntervalType, BinaryType,
|
StringType, DecimalType.defaultConcreteType, CalendarIntervalType, BinaryType,
|
||||||
|
@ -44,16 +44,18 @@ class MutableProjectionSuite extends SparkFunSuite with ExpressionEvalHelper {
|
||||||
|
|
||||||
testBothCodegenAndInterpreted("fixed-length types") {
|
testBothCodegenAndInterpreted("fixed-length types") {
|
||||||
val inputRow = InternalRow.fromSeq(Seq(
|
val inputRow = InternalRow.fromSeq(Seq(
|
||||||
true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L, Int.MaxValue) ++
|
true, 3.toByte, 15.toShort, -83, 129L, 1.0f, 5.0, 1, 2L) ++
|
||||||
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue))
|
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue) ++
|
||||||
|
Seq.tabulate(yearMonthIntervalTypes.length)(_ => Int.MaxValue))
|
||||||
val proj = createMutableProjection(fixedLengthTypes)
|
val proj = createMutableProjection(fixedLengthTypes)
|
||||||
assert(proj(inputRow) === inputRow)
|
assert(proj(inputRow) === inputRow)
|
||||||
}
|
}
|
||||||
|
|
||||||
testBothCodegenAndInterpreted("unsafe buffer") {
|
testBothCodegenAndInterpreted("unsafe buffer") {
|
||||||
val inputRow = InternalRow.fromSeq(Seq(
|
val inputRow = InternalRow.fromSeq(Seq(
|
||||||
false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L, Int.MinValue) ++
|
false, 1.toByte, 9.toShort, -18, 53L, 3.2f, 7.8, 4, 9L) ++
|
||||||
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue))
|
Seq.tabulate(dayTimeIntervalTypes.length)(_ => Long.MaxValue) ++
|
||||||
|
Seq.tabulate(yearMonthIntervalTypes.length)(_ => Int.MaxValue))
|
||||||
val numFields = fixedLengthTypes.length
|
val numFields = fixedLengthTypes.length
|
||||||
val numBytes = Platform.BYTE_ARRAY_OFFSET + UnsafeRow.calculateBitSetWidthInBytes(numFields) +
|
val numBytes = Platform.BYTE_ARRAY_OFFSET + UnsafeRow.calculateBitSetWidthInBytes(numFields) +
|
||||||
UnsafeRow.WORD_SIZE * numFields
|
UnsafeRow.WORD_SIZE * numFields
|
||||||
|
|
|
@ -275,8 +275,8 @@ class PushFoldableIntoBranchesSuite
|
||||||
Literal(new CalendarInterval(1, 2, 0)))),
|
Literal(new CalendarInterval(1, 2, 0)))),
|
||||||
If(a, Literal(Date.valueOf("2021-02-02")), Literal(Date.valueOf("2021-02-03"))))
|
If(a, Literal(Date.valueOf("2021-02-02")), Literal(Date.valueOf("2021-02-03"))))
|
||||||
assertEquivalent(DateAddYMInterval(Literal(d),
|
assertEquivalent(DateAddYMInterval(Literal(d),
|
||||||
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType),
|
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType()),
|
||||||
Literal.create(Period.ofMonths(2), YearMonthIntervalType))),
|
Literal.create(Period.ofMonths(2), YearMonthIntervalType()))),
|
||||||
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01"))))
|
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01"))))
|
||||||
assertEquivalent(DateDiff(Literal(d),
|
assertEquivalent(DateDiff(Literal(d),
|
||||||
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))),
|
If(a, Literal(Date.valueOf("2021-02-01")), Literal(Date.valueOf("2021-03-01")))),
|
||||||
|
@ -286,8 +286,8 @@ class PushFoldableIntoBranchesSuite
|
||||||
If(a, Literal(Date.valueOf("2020-12-31")), Literal(Date.valueOf("2020-12-30"))))
|
If(a, Literal(Date.valueOf("2020-12-31")), Literal(Date.valueOf("2020-12-30"))))
|
||||||
assertEquivalent(TimestampAddYMInterval(
|
assertEquivalent(TimestampAddYMInterval(
|
||||||
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
|
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
|
||||||
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType),
|
If(a, Literal.create(Period.ofMonths(1), YearMonthIntervalType()),
|
||||||
Literal.create(Period.ofMonths(2), YearMonthIntervalType))),
|
Literal.create(Period.ofMonths(2), YearMonthIntervalType()))),
|
||||||
If(a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType),
|
If(a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType),
|
||||||
Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType)))
|
Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType)))
|
||||||
assertEquivalent(TimeAdd(
|
assertEquivalent(TimeAdd(
|
||||||
|
@ -312,8 +312,8 @@ class PushFoldableIntoBranchesSuite
|
||||||
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-02"))),
|
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-02"))),
|
||||||
(c, Literal(Date.valueOf("2021-02-03")))), None))
|
(c, Literal(Date.valueOf("2021-02-03")))), None))
|
||||||
assertEquivalent(DateAddYMInterval(Literal(d),
|
assertEquivalent(DateAddYMInterval(Literal(d),
|
||||||
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)),
|
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType())),
|
||||||
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)),
|
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType()))), None)),
|
||||||
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
|
CaseWhen(Seq((a, Literal(Date.valueOf("2021-02-01"))),
|
||||||
(c, Literal(Date.valueOf("2021-03-01")))), None))
|
(c, Literal(Date.valueOf("2021-03-01")))), None))
|
||||||
assertEquivalent(DateDiff(Literal(d),
|
assertEquivalent(DateDiff(Literal(d),
|
||||||
|
@ -326,8 +326,8 @@ class PushFoldableIntoBranchesSuite
|
||||||
(c, Literal(Date.valueOf("2020-12-30")))), None))
|
(c, Literal(Date.valueOf("2020-12-30")))), None))
|
||||||
assertEquivalent(TimestampAddYMInterval(
|
assertEquivalent(TimestampAddYMInterval(
|
||||||
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
|
Literal.create(Timestamp.valueOf("2021-01-01 00:00:00.000"), TimestampType),
|
||||||
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType)),
|
CaseWhen(Seq((a, Literal.create(Period.ofMonths(1), YearMonthIntervalType())),
|
||||||
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType))), None)),
|
(c, Literal.create(Period.ofMonths(2), YearMonthIntervalType()))), None)),
|
||||||
CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType)),
|
CaseWhen(Seq((a, Literal.create(Timestamp.valueOf("2021-02-01 00:00:00"), TimestampType)),
|
||||||
(c, Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))), None))
|
(c, Literal.create(Timestamp.valueOf("2021-03-01 00:00:00"), TimestampType))), None))
|
||||||
assertEquivalent(TimeAdd(
|
assertEquivalent(TimeAdd(
|
||||||
|
|
|
@ -63,7 +63,7 @@ class DataTypeParserSuite extends SparkFunSuite {
|
||||||
checkDataType("BINARY", BinaryType)
|
checkDataType("BINARY", BinaryType)
|
||||||
checkDataType("void", NullType)
|
checkDataType("void", NullType)
|
||||||
checkDataType("interval", CalendarIntervalType)
|
checkDataType("interval", CalendarIntervalType)
|
||||||
checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType)
|
checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType())
|
||||||
checkDataType("interval day to second", DayTimeIntervalType())
|
checkDataType("interval day to second", DayTimeIntervalType())
|
||||||
|
|
||||||
checkDataType("array<doublE>", ArrayType(DoubleType, true))
|
checkDataType("array<doublE>", ArrayType(DoubleType, true))
|
||||||
|
|
|
@ -503,6 +503,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-35016: format year-month intervals") {
|
test("SPARK-35016: format year-month intervals") {
|
||||||
|
import org.apache.spark.sql.types.YearMonthIntervalType._
|
||||||
Seq(
|
Seq(
|
||||||
0 -> ("0-0", "INTERVAL '0-0' YEAR TO MONTH"),
|
0 -> ("0-0", "INTERVAL '0-0' YEAR TO MONTH"),
|
||||||
-11 -> ("-0-11", "INTERVAL '-0-11' YEAR TO MONTH"),
|
-11 -> ("-0-11", "INTERVAL '-0-11' YEAR TO MONTH"),
|
||||||
|
@ -514,8 +515,8 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
|
||||||
Int.MinValue -> ("-178956970-8", "INTERVAL '-178956970-8' YEAR TO MONTH"),
|
Int.MinValue -> ("-178956970-8", "INTERVAL '-178956970-8' YEAR TO MONTH"),
|
||||||
Int.MaxValue -> ("178956970-7", "INTERVAL '178956970-7' YEAR TO MONTH")
|
Int.MaxValue -> ("178956970-7", "INTERVAL '178956970-7' YEAR TO MONTH")
|
||||||
).foreach { case (months, (hiveIntervalStr, ansiIntervalStr)) =>
|
).foreach { case (months, (hiveIntervalStr, ansiIntervalStr)) =>
|
||||||
assert(toYearMonthIntervalString(months, ANSI_STYLE) === ansiIntervalStr)
|
assert(toYearMonthIntervalString(months, ANSI_STYLE, YEAR, MONTH) === ansiIntervalStr)
|
||||||
assert(toYearMonthIntervalString(months, HIVE_STYLE) === hiveIntervalStr)
|
assert(toYearMonthIntervalString(months, HIVE_STYLE, YEAR, MONTH) === hiveIntervalStr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ import com.fasterxml.jackson.core.JsonParseException
|
||||||
import org.apache.spark.{SparkException, SparkFunSuite}
|
import org.apache.spark.{SparkException, SparkFunSuite}
|
||||||
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
|
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
|
||||||
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
|
import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
|
||||||
import org.apache.spark.sql.types.DataTypeTestUtils.dayTimeIntervalTypes
|
import org.apache.spark.sql.types.DataTypeTestUtils.{dayTimeIntervalTypes, yearMonthIntervalTypes}
|
||||||
|
|
||||||
class DataTypeSuite extends SparkFunSuite {
|
class DataTypeSuite extends SparkFunSuite {
|
||||||
|
|
||||||
|
@ -256,7 +256,7 @@ class DataTypeSuite extends SparkFunSuite {
|
||||||
checkDataTypeFromJson(VarcharType(10))
|
checkDataTypeFromJson(VarcharType(10))
|
||||||
checkDataTypeFromDDL(VarcharType(11))
|
checkDataTypeFromDDL(VarcharType(11))
|
||||||
|
|
||||||
checkDataTypeFromDDL(YearMonthIntervalType)
|
checkDataTypeFromDDL(YearMonthIntervalType())
|
||||||
dayTimeIntervalTypes.foreach(checkDataTypeFromDDL)
|
dayTimeIntervalTypes.foreach(checkDataTypeFromDDL)
|
||||||
|
|
||||||
val metadata = new MetadataBuilder()
|
val metadata = new MetadataBuilder()
|
||||||
|
@ -325,7 +325,7 @@ class DataTypeSuite extends SparkFunSuite {
|
||||||
checkDefaultSize(CharType(100), 100)
|
checkDefaultSize(CharType(100), 100)
|
||||||
checkDefaultSize(VarcharType(5), 5)
|
checkDefaultSize(VarcharType(5), 5)
|
||||||
checkDefaultSize(VarcharType(10), 10)
|
checkDefaultSize(VarcharType(10), 10)
|
||||||
checkDefaultSize(YearMonthIntervalType, 4)
|
yearMonthIntervalTypes.foreach(checkDefaultSize(_, 4))
|
||||||
dayTimeIntervalTypes.foreach(checkDefaultSize(_, 8))
|
dayTimeIntervalTypes.foreach(checkDefaultSize(_, 8))
|
||||||
|
|
||||||
def checkEqualsIgnoreCompatibleNullability(
|
def checkEqualsIgnoreCompatibleNullability(
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
package org.apache.spark.sql.types
|
package org.apache.spark.sql.types
|
||||||
|
|
||||||
import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, MINUTE, SECOND}
|
import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, MINUTE, SECOND}
|
||||||
|
import org.apache.spark.sql.types.YearMonthIntervalType.{MONTH, YEAR}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility functions for working with DataTypes in tests.
|
* Utility functions for working with DataTypes in tests.
|
||||||
|
@ -63,12 +64,16 @@ object DataTypeTestUtils {
|
||||||
DayTimeIntervalType(MINUTE, SECOND),
|
DayTimeIntervalType(MINUTE, SECOND),
|
||||||
DayTimeIntervalType(SECOND, SECOND))
|
DayTimeIntervalType(SECOND, SECOND))
|
||||||
|
|
||||||
|
val yearMonthIntervalTypes: Seq[YearMonthIntervalType] = Seq(
|
||||||
|
YearMonthIntervalType(YEAR, MONTH),
|
||||||
|
YearMonthIntervalType(YEAR, YEAR),
|
||||||
|
YearMonthIntervalType(MONTH, MONTH))
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Instances of all [[NumericType]]s and [[CalendarIntervalType]]
|
* Instances of all [[NumericType]]s and [[CalendarIntervalType]]
|
||||||
*/
|
*/
|
||||||
val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal ++ Set(
|
val numericAndInterval: Set[DataType] = numericTypeWithoutDecimal ++
|
||||||
CalendarIntervalType,
|
Set(CalendarIntervalType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
|
||||||
YearMonthIntervalType) ++ dayTimeIntervalTypes
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All the types that support ordering
|
* All the types that support ordering
|
||||||
|
@ -79,8 +84,7 @@ object DataTypeTestUtils {
|
||||||
TimestampWithoutTZType,
|
TimestampWithoutTZType,
|
||||||
DateType,
|
DateType,
|
||||||
StringType,
|
StringType,
|
||||||
BinaryType,
|
BinaryType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
|
||||||
YearMonthIntervalType) ++ dayTimeIntervalTypes
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* All the types that we can use in a property check
|
* All the types that we can use in a property check
|
||||||
|
@ -96,9 +100,7 @@ object DataTypeTestUtils {
|
||||||
DateType,
|
DateType,
|
||||||
StringType,
|
StringType,
|
||||||
TimestampType,
|
TimestampType,
|
||||||
TimestampWithoutTZType,
|
TimestampWithoutTZType) ++ dayTimeIntervalTypes ++ yearMonthIntervalTypes
|
||||||
YearMonthIntervalType
|
|
||||||
) ++ dayTimeIntervalTypes
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Instances of [[ArrayType]] for all [[AtomicType]]s. Arrays of these types may contain null.
|
* Instances of [[ArrayType]] for all [[AtomicType]]s. Arrays of these types may contain null.
|
||||||
|
|
|
@ -48,7 +48,7 @@ class ArrowUtilsSuite extends SparkFunSuite {
|
||||||
roundtrip(BinaryType)
|
roundtrip(BinaryType)
|
||||||
roundtrip(DecimalType.SYSTEM_DEFAULT)
|
roundtrip(DecimalType.SYSTEM_DEFAULT)
|
||||||
roundtrip(DateType)
|
roundtrip(DateType)
|
||||||
roundtrip(YearMonthIntervalType)
|
roundtrip(YearMonthIntervalType())
|
||||||
roundtrip(DayTimeIntervalType())
|
roundtrip(DayTimeIntervalType())
|
||||||
val tsExMsg = intercept[UnsupportedOperationException] {
|
val tsExMsg = intercept[UnsupportedOperationException] {
|
||||||
roundtrip(TimestampType)
|
roundtrip(TimestampType)
|
||||||
|
|
|
@ -117,8 +117,8 @@ object HiveResult {
|
||||||
struct.toSeq.zip(fields).map { case (v, t) =>
|
struct.toSeq.zip(fields).map { case (v, t) =>
|
||||||
s""""${t.name}":${toHiveString((v, t.dataType), true, formatters)}"""
|
s""""${t.name}":${toHiveString((v, t.dataType), true, formatters)}"""
|
||||||
}.mkString("{", ",", "}")
|
}.mkString("{", ",", "}")
|
||||||
case (period: Period, YearMonthIntervalType) =>
|
case (period: Period, YearMonthIntervalType(startField, endField)) =>
|
||||||
toYearMonthIntervalString(periodToMonths(period), HIVE_STYLE)
|
toYearMonthIntervalString(periodToMonths(period), HIVE_STYLE, startField, endField)
|
||||||
case (duration: Duration, DayTimeIntervalType(startField, endField)) =>
|
case (duration: Duration, DayTimeIntervalType(startField, endField)) =>
|
||||||
toDayTimeIntervalString(durationToMicros(duration), HIVE_STYLE, startField, endField)
|
toDayTimeIntervalString(durationToMicros(duration), HIVE_STYLE, startField, endField)
|
||||||
case (other, _: UserDefinedType[_]) => other.toString
|
case (other, _: UserDefinedType[_]) => other.toString
|
||||||
|
|
|
@ -158,7 +158,8 @@ abstract class HashMapGenerator(
|
||||||
|
|
||||||
dataType match {
|
dataType match {
|
||||||
case BooleanType => hashInt(s"$input ? 1 : 0")
|
case BooleanType => hashInt(s"$input ? 1 : 0")
|
||||||
case ByteType | ShortType | IntegerType | DateType | YearMonthIntervalType => hashInt(input)
|
case ByteType | ShortType | IntegerType | DateType | _: YearMonthIntervalType =>
|
||||||
|
hashInt(input)
|
||||||
case LongType | TimestampType | _: DayTimeIntervalType => hashLong(input)
|
case LongType | TimestampType | _: DayTimeIntervalType => hashLong(input)
|
||||||
case FloatType => hashInt(s"Float.floatToIntBits($input)")
|
case FloatType => hashInt(s"Float.floatToIntBits($input)")
|
||||||
case DoubleType => hashLong(s"Double.doubleToLongBits($input)")
|
case DoubleType => hashLong(s"Double.doubleToLongBits($input)")
|
||||||
|
|
|
@ -87,7 +87,7 @@ sealed trait BufferSetterGetterUtils {
|
||||||
(row: InternalRow, ordinal: Int) =>
|
(row: InternalRow, ordinal: Int) =>
|
||||||
if (row.isNullAt(ordinal)) null else row.getLong(ordinal)
|
if (row.isNullAt(ordinal)) null else row.getLong(ordinal)
|
||||||
|
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
(row: InternalRow, ordinal: Int) =>
|
(row: InternalRow, ordinal: Int) =>
|
||||||
if (row.isNullAt(ordinal)) null else row.getInt(ordinal)
|
if (row.isNullAt(ordinal)) null else row.getInt(ordinal)
|
||||||
|
|
||||||
|
@ -195,7 +195,7 @@ sealed trait BufferSetterGetterUtils {
|
||||||
row.setNullAt(ordinal)
|
row.setNullAt(ordinal)
|
||||||
}
|
}
|
||||||
|
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
(row: InternalRow, ordinal: Int, value: Any) =>
|
(row: InternalRow, ordinal: Int, value: Any) =>
|
||||||
if (value != null) {
|
if (value != null) {
|
||||||
row.setInt(ordinal, value.asInstanceOf[Int])
|
row.setInt(ordinal, value.asInstanceOf[Int])
|
||||||
|
|
|
@ -75,7 +75,7 @@ object ArrowWriter {
|
||||||
}
|
}
|
||||||
new StructWriter(vector, children.toArray)
|
new StructWriter(vector, children.toArray)
|
||||||
case (NullType, vector: NullVector) => new NullWriter(vector)
|
case (NullType, vector: NullVector) => new NullWriter(vector)
|
||||||
case (YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector)
|
case (_: YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector)
|
||||||
case (_: DayTimeIntervalType, vector: IntervalDayVector) => new IntervalDayWriter(vector)
|
case (_: DayTimeIntervalType, vector: IntervalDayVector) => new IntervalDayWriter(vector)
|
||||||
case (dt, _) =>
|
case (dt, _) =>
|
||||||
throw QueryExecutionErrors.unsupportedDataTypeError(dt)
|
throw QueryExecutionErrors.unsupportedDataTypeError(dt)
|
||||||
|
|
|
@ -139,7 +139,7 @@ private[sql] object ColumnAccessor {
|
||||||
case BooleanType => new BooleanColumnAccessor(buf)
|
case BooleanType => new BooleanColumnAccessor(buf)
|
||||||
case ByteType => new ByteColumnAccessor(buf)
|
case ByteType => new ByteColumnAccessor(buf)
|
||||||
case ShortType => new ShortColumnAccessor(buf)
|
case ShortType => new ShortColumnAccessor(buf)
|
||||||
case IntegerType | DateType | YearMonthIntervalType => new IntColumnAccessor(buf)
|
case IntegerType | DateType | _: YearMonthIntervalType => new IntColumnAccessor(buf)
|
||||||
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnAccessor(buf)
|
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnAccessor(buf)
|
||||||
case FloatType => new FloatColumnAccessor(buf)
|
case FloatType => new FloatColumnAccessor(buf)
|
||||||
case DoubleType => new DoubleColumnAccessor(buf)
|
case DoubleType => new DoubleColumnAccessor(buf)
|
||||||
|
|
|
@ -174,7 +174,7 @@ private[columnar] object ColumnBuilder {
|
||||||
case BooleanType => new BooleanColumnBuilder
|
case BooleanType => new BooleanColumnBuilder
|
||||||
case ByteType => new ByteColumnBuilder
|
case ByteType => new ByteColumnBuilder
|
||||||
case ShortType => new ShortColumnBuilder
|
case ShortType => new ShortColumnBuilder
|
||||||
case IntegerType | DateType | YearMonthIntervalType => new IntColumnBuilder
|
case IntegerType | DateType | _: YearMonthIntervalType => new IntColumnBuilder
|
||||||
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnBuilder
|
case LongType | TimestampType | _: DayTimeIntervalType => new LongColumnBuilder
|
||||||
case FloatType => new FloatColumnBuilder
|
case FloatType => new FloatColumnBuilder
|
||||||
case DoubleType => new DoubleColumnBuilder
|
case DoubleType => new DoubleColumnBuilder
|
||||||
|
|
|
@ -257,7 +257,7 @@ private[columnar] object LONG extends NativeColumnType(LongType, 8) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private[columnar] object YEAR_MONTH_INTERVAL extends NativeColumnType(YearMonthIntervalType, 4) {
|
private[columnar] object YEAR_MONTH_INTERVAL extends NativeColumnType(YearMonthIntervalType(), 4) {
|
||||||
override def append(v: Int, buffer: ByteBuffer): Unit = {
|
override def append(v: Int, buffer: ByteBuffer): Unit = {
|
||||||
buffer.putInt(v)
|
buffer.putInt(v)
|
||||||
}
|
}
|
||||||
|
@ -817,7 +817,7 @@ private[columnar] object ColumnType {
|
||||||
case BooleanType => BOOLEAN
|
case BooleanType => BOOLEAN
|
||||||
case ByteType => BYTE
|
case ByteType => BYTE
|
||||||
case ShortType => SHORT
|
case ShortType => SHORT
|
||||||
case IntegerType | DateType | YearMonthIntervalType => INT
|
case IntegerType | DateType | _: YearMonthIntervalType => INT
|
||||||
case LongType | TimestampType | _: DayTimeIntervalType => LONG
|
case LongType | TimestampType | _: DayTimeIntervalType => LONG
|
||||||
case FloatType => FLOAT
|
case FloatType => FLOAT
|
||||||
case DoubleType => DOUBLE
|
case DoubleType => DOUBLE
|
||||||
|
|
|
@ -80,7 +80,7 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
|
||||||
case BooleanType => classOf[BooleanColumnAccessor].getName
|
case BooleanType => classOf[BooleanColumnAccessor].getName
|
||||||
case ByteType => classOf[ByteColumnAccessor].getName
|
case ByteType => classOf[ByteColumnAccessor].getName
|
||||||
case ShortType => classOf[ShortColumnAccessor].getName
|
case ShortType => classOf[ShortColumnAccessor].getName
|
||||||
case IntegerType | DateType | YearMonthIntervalType => classOf[IntColumnAccessor].getName
|
case IntegerType | DateType | _: YearMonthIntervalType => classOf[IntColumnAccessor].getName
|
||||||
case LongType | TimestampType | _: DayTimeIntervalType =>
|
case LongType | TimestampType | _: DayTimeIntervalType =>
|
||||||
classOf[LongColumnAccessor].getName
|
classOf[LongColumnAccessor].getName
|
||||||
case FloatType => classOf[FloatColumnAccessor].getName
|
case FloatType => classOf[FloatColumnAccessor].getName
|
||||||
|
|
|
@ -95,9 +95,9 @@ trait WindowExecBase extends UnaryExecNode {
|
||||||
// Create the projection which returns the current 'value' modified by adding the offset.
|
// Create the projection which returns the current 'value' modified by adding the offset.
|
||||||
val boundExpr = (expr.dataType, boundOffset.dataType) match {
|
val boundExpr = (expr.dataType, boundOffset.dataType) match {
|
||||||
case (DateType, IntegerType) => DateAdd(expr, boundOffset)
|
case (DateType, IntegerType) => DateAdd(expr, boundOffset)
|
||||||
case (DateType, YearMonthIntervalType) => DateAddYMInterval(expr, boundOffset)
|
case (DateType, _: YearMonthIntervalType) => DateAddYMInterval(expr, boundOffset)
|
||||||
case (TimestampType, CalendarIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
|
case (TimestampType, CalendarIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
|
||||||
case (TimestampType, YearMonthIntervalType) =>
|
case (TimestampType, _: YearMonthIntervalType) =>
|
||||||
TimestampAddYMInterval(expr, boundOffset, Some(timeZone))
|
TimestampAddYMInterval(expr, boundOffset, Some(timeZone))
|
||||||
case (TimestampType, _: DayTimeIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
|
case (TimestampType, _: DayTimeIntervalType) => TimeAdd(expr, boundOffset, Some(timeZone))
|
||||||
case (a, b) if a == b => Add(expr, boundOffset)
|
case (a, b) if a == b => Add(expr, boundOffset)
|
||||||
|
|
|
@ -1129,7 +1129,9 @@ class DataFrameAggregateSuite extends QueryTest
|
||||||
val sumDF = df.select(sum($"year-month"), sum($"day-time"))
|
val sumDF = df.select(sum($"year-month"), sum($"day-time"))
|
||||||
checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
|
checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
|
||||||
assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
||||||
assert(sumDF.schema == StructType(Seq(StructField("sum(year-month)", YearMonthIntervalType),
|
assert(sumDF.schema == StructType(Seq(
|
||||||
|
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
|
||||||
|
StructField("sum(year-month)", YearMonthIntervalType()),
|
||||||
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
||||||
StructField("sum(day-time)", DayTimeIntervalType()))))
|
StructField("sum(day-time)", DayTimeIntervalType()))))
|
||||||
|
|
||||||
|
@ -1139,7 +1141,8 @@ class DataFrameAggregateSuite extends QueryTest
|
||||||
Row(3, Period.of(1, 6, 0), Duration.ofDays(-11)) :: Nil)
|
Row(3, Period.of(1, 6, 0), Duration.ofDays(-11)) :: Nil)
|
||||||
assert(find(sumDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
assert(find(sumDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
||||||
assert(sumDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
|
assert(sumDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
|
||||||
StructField("sum(year-month)", YearMonthIntervalType),
|
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
|
||||||
|
StructField("sum(year-month)", YearMonthIntervalType()),
|
||||||
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
||||||
StructField("sum(day-time)", DayTimeIntervalType()))))
|
StructField("sum(day-time)", DayTimeIntervalType()))))
|
||||||
|
|
||||||
|
@ -1169,7 +1172,9 @@ class DataFrameAggregateSuite extends QueryTest
|
||||||
val avgDF = df.select(avg($"year-month"), avg($"day-time"))
|
val avgDF = df.select(avg($"year-month"), avg($"day-time"))
|
||||||
checkAnswer(avgDF, Row(Period.ofMonths(7), Duration.ofDays(0)))
|
checkAnswer(avgDF, Row(Period.ofMonths(7), Duration.ofDays(0)))
|
||||||
assert(find(avgDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
assert(find(avgDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
||||||
assert(avgDF.schema == StructType(Seq(StructField("avg(year-month)", YearMonthIntervalType),
|
assert(avgDF.schema == StructType(Seq(
|
||||||
|
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
|
||||||
|
StructField("avg(year-month)", YearMonthIntervalType()),
|
||||||
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
||||||
StructField("avg(day-time)", DayTimeIntervalType()))))
|
StructField("avg(day-time)", DayTimeIntervalType()))))
|
||||||
|
|
||||||
|
@ -1179,7 +1184,8 @@ class DataFrameAggregateSuite extends QueryTest
|
||||||
Row(3, Period.ofMonths(9), Duration.ofDays(-5).plusHours(-12)) :: Nil)
|
Row(3, Period.ofMonths(9), Duration.ofDays(-5).plusHours(-12)) :: Nil)
|
||||||
assert(find(avgDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
assert(find(avgDF2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
|
||||||
assert(avgDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
|
assert(avgDF2.schema == StructType(Seq(StructField("class", IntegerType, false),
|
||||||
StructField("avg(year-month)", YearMonthIntervalType),
|
// TODO(SPARK-35775): Check all year-month interval types in aggregate expressions
|
||||||
|
StructField("avg(year-month)", YearMonthIntervalType()),
|
||||||
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
// TODO(SPARK-35729): Check all day-time interval types in aggregate expressions
|
||||||
StructField("avg(day-time)", DayTimeIntervalType()))))
|
StructField("avg(day-time)", DayTimeIntervalType()))))
|
||||||
|
|
||||||
|
|
|
@ -908,18 +908,21 @@ class UDFSuite extends QueryTest with SharedSparkSession {
|
||||||
val incMonth = udf((p: java.time.Period) => p.plusMonths(1))
|
val incMonth = udf((p: java.time.Period) => p.plusMonths(1))
|
||||||
val result = input.select(incMonth($"p").as("new_p"))
|
val result = input.select(incMonth($"p").as("new_p"))
|
||||||
checkAnswer(result, Row(java.time.Period.ofYears(1)) :: Nil)
|
checkAnswer(result, Row(java.time.Period.ofYears(1)) :: Nil)
|
||||||
assert(result.schema === new StructType().add("new_p", YearMonthIntervalType))
|
// TODO(SPARK-35777): Check all year-month interval types in UDF
|
||||||
|
assert(result.schema === new StructType().add("new_p", YearMonthIntervalType()))
|
||||||
// UDF produces `null`
|
// UDF produces `null`
|
||||||
val nullFunc = udf((_: java.time.Period) => null.asInstanceOf[java.time.Period])
|
val nullFunc = udf((_: java.time.Period) => null.asInstanceOf[java.time.Period])
|
||||||
val nullResult = input.select(nullFunc($"p").as("null_p"))
|
val nullResult = input.select(nullFunc($"p").as("null_p"))
|
||||||
checkAnswer(nullResult, Row(null) :: Nil)
|
checkAnswer(nullResult, Row(null) :: Nil)
|
||||||
assert(nullResult.schema === new StructType().add("null_p", YearMonthIntervalType))
|
// TODO(SPARK-35777): Check all year-month interval types in UDF
|
||||||
|
assert(nullResult.schema === new StructType().add("null_p", YearMonthIntervalType()))
|
||||||
// Input parameter of UDF is null
|
// Input parameter of UDF is null
|
||||||
val nullInput = Seq(null.asInstanceOf[java.time.Period]).toDF("null_p")
|
val nullInput = Seq(null.asInstanceOf[java.time.Period]).toDF("null_p")
|
||||||
val constPeriod = udf((_: java.time.Period) => java.time.Period.ofYears(10))
|
val constPeriod = udf((_: java.time.Period) => java.time.Period.ofYears(10))
|
||||||
val constResult = nullInput.select(constPeriod($"null_p").as("10_years"))
|
val constResult = nullInput.select(constPeriod($"null_p").as("10_years"))
|
||||||
checkAnswer(constResult, Row(java.time.Period.ofYears(10)) :: Nil)
|
checkAnswer(constResult, Row(java.time.Period.ofYears(10)) :: Nil)
|
||||||
assert(constResult.schema === new StructType().add("10_years", YearMonthIntervalType))
|
// TODO(SPARK-35777): Check all year-month interval types in UDF
|
||||||
|
assert(constResult.schema === new StructType().add("10_years", YearMonthIntervalType()))
|
||||||
// Error in the conversion of UDF result to the internal representation of year-month interval
|
// Error in the conversion of UDF result to the internal representation of year-month interval
|
||||||
val overflowFunc = udf((p: java.time.Period) => p.plusYears(Long.MaxValue))
|
val overflowFunc = udf((p: java.time.Period) => p.plusYears(Long.MaxValue))
|
||||||
val e = intercept[SparkException] {
|
val e = intercept[SparkException] {
|
||||||
|
|
|
@ -56,7 +56,7 @@ class ArrowWriterSuite extends SparkFunSuite {
|
||||||
case BinaryType => reader.getBinary(rowId)
|
case BinaryType => reader.getBinary(rowId)
|
||||||
case DateType => reader.getInt(rowId)
|
case DateType => reader.getInt(rowId)
|
||||||
case TimestampType => reader.getLong(rowId)
|
case TimestampType => reader.getLong(rowId)
|
||||||
case YearMonthIntervalType => reader.getInt(rowId)
|
case _: YearMonthIntervalType => reader.getInt(rowId)
|
||||||
case _: DayTimeIntervalType => reader.getLong(rowId)
|
case _: DayTimeIntervalType => reader.getLong(rowId)
|
||||||
}
|
}
|
||||||
assert(value === datum)
|
assert(value === datum)
|
||||||
|
@ -77,7 +77,7 @@ class ArrowWriterSuite extends SparkFunSuite {
|
||||||
check(DateType, Seq(0, 1, 2, null, 4))
|
check(DateType, Seq(0, 1, 2, null, 4))
|
||||||
check(TimestampType, Seq(0L, 3.6e9.toLong, null, 8.64e10.toLong), "America/Los_Angeles")
|
check(TimestampType, Seq(0L, 3.6e9.toLong, null, 8.64e10.toLong), "America/Los_Angeles")
|
||||||
check(NullType, Seq(null, null, null))
|
check(NullType, Seq(null, null, null))
|
||||||
check(YearMonthIntervalType, Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue))
|
check(YearMonthIntervalType(), Seq(null, 0, 1, -1, Int.MaxValue, Int.MinValue))
|
||||||
check(DayTimeIntervalType(), Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L),
|
check(DayTimeIntervalType(), Seq(null, 0L, 1000L, -1000L, (Long.MaxValue - 807L),
|
||||||
(Long.MinValue + 808L)))
|
(Long.MinValue + 808L)))
|
||||||
}
|
}
|
||||||
|
@ -128,7 +128,7 @@ class ArrowWriterSuite extends SparkFunSuite {
|
||||||
case DoubleType => reader.getDoubles(0, data.size)
|
case DoubleType => reader.getDoubles(0, data.size)
|
||||||
case DateType => reader.getInts(0, data.size)
|
case DateType => reader.getInts(0, data.size)
|
||||||
case TimestampType => reader.getLongs(0, data.size)
|
case TimestampType => reader.getLongs(0, data.size)
|
||||||
case YearMonthIntervalType => reader.getInts(0, data.size)
|
case _: YearMonthIntervalType => reader.getInts(0, data.size)
|
||||||
case _: DayTimeIntervalType => reader.getLongs(0, data.size)
|
case _: DayTimeIntervalType => reader.getLongs(0, data.size)
|
||||||
}
|
}
|
||||||
assert(values === data)
|
assert(values === data)
|
||||||
|
@ -144,7 +144,8 @@ class ArrowWriterSuite extends SparkFunSuite {
|
||||||
check(DoubleType, (0 until 10).map(_.toDouble))
|
check(DoubleType, (0 until 10).map(_.toDouble))
|
||||||
check(DateType, (0 until 10))
|
check(DateType, (0 until 10))
|
||||||
check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles")
|
check(TimestampType, (0 until 10).map(_ * 4.32e10.toLong), "America/Los_Angeles")
|
||||||
check(YearMonthIntervalType, (0 until 10))
|
// TODO(SPARK-35776): Check all year-month interval types in arrow
|
||||||
|
check(YearMonthIntervalType(), (0 until 10))
|
||||||
// TODO(SPARK-35731): Check all day-time interval types in arrow
|
// TODO(SPARK-35731): Check all day-time interval types in arrow
|
||||||
check(DayTimeIntervalType(), (-10 until 10).map(_ * 1000.toLong))
|
check(DayTimeIntervalType(), (-10 until 10).map(_ * 1000.toLong))
|
||||||
}
|
}
|
||||||
|
|
|
@ -121,7 +121,7 @@ private[hive] class SparkExecuteStatementOperation(
|
||||||
false,
|
false,
|
||||||
timeFormatters)
|
timeFormatters)
|
||||||
case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] |
|
case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] |
|
||||||
YearMonthIntervalType | _: DayTimeIntervalType =>
|
_: YearMonthIntervalType | _: DayTimeIntervalType =>
|
||||||
to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters)
|
to += toHiveString((from.get(ordinal), dataTypes(ordinal)), false, timeFormatters)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -377,7 +377,7 @@ object SparkExecuteStatementOperation {
|
||||||
val attrTypeString = field.dataType match {
|
val attrTypeString = field.dataType match {
|
||||||
case NullType => "void"
|
case NullType => "void"
|
||||||
case CalendarIntervalType => StringType.catalogString
|
case CalendarIntervalType => StringType.catalogString
|
||||||
case YearMonthIntervalType => "interval_year_month"
|
case _: YearMonthIntervalType => "interval_year_month"
|
||||||
case _: DayTimeIntervalType => "interval_day_time"
|
case _: DayTimeIntervalType => "interval_day_time"
|
||||||
case other => other.catalogString
|
case other => other.catalogString
|
||||||
}
|
}
|
||||||
|
|
|
@ -131,7 +131,8 @@ private[hive] class SparkGetColumnsOperation(
|
||||||
*/
|
*/
|
||||||
private def getColumnSize(typ: DataType): Option[Int] = typ match {
|
private def getColumnSize(typ: DataType): Option[Int] = typ match {
|
||||||
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
|
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
|
||||||
CalendarIntervalType | NullType | YearMonthIntervalType | _: DayTimeIntervalType) =>
|
CalendarIntervalType | NullType |
|
||||||
|
_: YearMonthIntervalType | _: DayTimeIntervalType) =>
|
||||||
Some(dt.defaultSize)
|
Some(dt.defaultSize)
|
||||||
case CharType(n) => Some(n)
|
case CharType(n) => Some(n)
|
||||||
case StructType(fields) =>
|
case StructType(fields) =>
|
||||||
|
@ -186,7 +187,7 @@ private[hive] class SparkGetColumnsOperation(
|
||||||
case _: MapType => java.sql.Types.JAVA_OBJECT
|
case _: MapType => java.sql.Types.JAVA_OBJECT
|
||||||
case _: StructType => java.sql.Types.STRUCT
|
case _: StructType => java.sql.Types.STRUCT
|
||||||
// Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
|
// Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
|
||||||
case _: CalendarIntervalType | YearMonthIntervalType | _: DayTimeIntervalType =>
|
case _: CalendarIntervalType | _: YearMonthIntervalType | _: DayTimeIntervalType =>
|
||||||
java.sql.Types.OTHER
|
java.sql.Types.OTHER
|
||||||
case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
|
case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
|
||||||
}
|
}
|
||||||
|
|
|
@ -392,8 +392,8 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase {
|
||||||
assert(rowSet.getString("TABLE_NAME") === viewName1)
|
assert(rowSet.getString("TABLE_NAME") === viewName1)
|
||||||
assert(rowSet.getString("COLUMN_NAME") === "i")
|
assert(rowSet.getString("COLUMN_NAME") === "i")
|
||||||
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
|
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
|
||||||
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql))
|
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType().sql))
|
||||||
assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize)
|
assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType().defaultSize)
|
||||||
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
|
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
|
||||||
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
|
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
|
||||||
assert(rowSet.getInt("NULLABLE") === 0)
|
assert(rowSet.getInt("NULLABLE") === 0)
|
||||||
|
|
|
@ -842,7 +842,7 @@ private[hive] trait HiveInspectors {
|
||||||
case TimestampType => PrimitiveObjectInspectorFactory.javaTimestampObjectInspector
|
case TimestampType => PrimitiveObjectInspectorFactory.javaTimestampObjectInspector
|
||||||
case _: DayTimeIntervalType =>
|
case _: DayTimeIntervalType =>
|
||||||
PrimitiveObjectInspectorFactory.javaHiveIntervalDayTimeObjectInspector
|
PrimitiveObjectInspectorFactory.javaHiveIntervalDayTimeObjectInspector
|
||||||
case YearMonthIntervalType =>
|
case _: YearMonthIntervalType =>
|
||||||
PrimitiveObjectInspectorFactory.javaHiveIntervalYearMonthObjectInspector
|
PrimitiveObjectInspectorFactory.javaHiveIntervalYearMonthObjectInspector
|
||||||
// TODO decimal precision?
|
// TODO decimal precision?
|
||||||
case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector
|
case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector
|
||||||
|
@ -891,7 +891,7 @@ private[hive] trait HiveInspectors {
|
||||||
getPrimitiveNullWritableConstantObjectInspector
|
getPrimitiveNullWritableConstantObjectInspector
|
||||||
case Literal(_, _: DayTimeIntervalType) =>
|
case Literal(_, _: DayTimeIntervalType) =>
|
||||||
getHiveIntervalDayTimeWritableConstantObjectInspector
|
getHiveIntervalDayTimeWritableConstantObjectInspector
|
||||||
case Literal(_, YearMonthIntervalType) =>
|
case Literal(_, _: YearMonthIntervalType) =>
|
||||||
getHiveIntervalYearMonthWritableConstantObjectInspector
|
getHiveIntervalYearMonthWritableConstantObjectInspector
|
||||||
case Literal(value, ArrayType(dt, _)) =>
|
case Literal(value, ArrayType(dt, _)) =>
|
||||||
val listObjectInspector = toInspector(dt)
|
val listObjectInspector = toInspector(dt)
|
||||||
|
@ -971,8 +971,8 @@ private[hive] trait HiveInspectors {
|
||||||
case _: JavaTimestampObjectInspector => TimestampType
|
case _: JavaTimestampObjectInspector => TimestampType
|
||||||
case _: WritableHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
|
case _: WritableHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
|
||||||
case _: JavaHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
|
case _: JavaHiveIntervalDayTimeObjectInspector => DayTimeIntervalType()
|
||||||
case _: WritableHiveIntervalYearMonthObjectInspector => YearMonthIntervalType
|
case _: WritableHiveIntervalYearMonthObjectInspector => YearMonthIntervalType()
|
||||||
case _: JavaHiveIntervalYearMonthObjectInspector => YearMonthIntervalType
|
case _: JavaHiveIntervalYearMonthObjectInspector => YearMonthIntervalType()
|
||||||
case _: WritableVoidObjectInspector => NullType
|
case _: WritableVoidObjectInspector => NullType
|
||||||
case _: JavaVoidObjectInspector => NullType
|
case _: JavaVoidObjectInspector => NullType
|
||||||
}
|
}
|
||||||
|
@ -1156,7 +1156,7 @@ private[hive] trait HiveInspectors {
|
||||||
case TimestampType => timestampTypeInfo
|
case TimestampType => timestampTypeInfo
|
||||||
case NullType => voidTypeInfo
|
case NullType => voidTypeInfo
|
||||||
case _: DayTimeIntervalType => intervalDayTimeTypeInfo
|
case _: DayTimeIntervalType => intervalDayTimeTypeInfo
|
||||||
case YearMonthIntervalType => intervalYearMonthTypeInfo
|
case _: YearMonthIntervalType => intervalYearMonthTypeInfo
|
||||||
case dt =>
|
case dt =>
|
||||||
throw new AnalysisException(
|
throw new AnalysisException(
|
||||||
s"${dt.catalogString} cannot be converted to Hive TypeInfo")
|
s"${dt.catalogString} cannot be converted to Hive TypeInfo")
|
||||||
|
|
|
@ -546,7 +546,8 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
|
||||||
AttributeReference("a", DayTimeIntervalType())(),
|
AttributeReference("a", DayTimeIntervalType())(),
|
||||||
AttributeReference("b", DayTimeIntervalType())(),
|
AttributeReference("b", DayTimeIntervalType())(),
|
||||||
AttributeReference("c", DayTimeIntervalType())(),
|
AttributeReference("c", DayTimeIntervalType())(),
|
||||||
AttributeReference("d", YearMonthIntervalType)()),
|
// TODO(SPARK-35772): Check all year-month interval types in HiveInspectors tests
|
||||||
|
AttributeReference("d", YearMonthIntervalType())()),
|
||||||
child = child,
|
child = child,
|
||||||
ioschema = hiveIOSchema),
|
ioschema = hiveIOSchema),
|
||||||
df.select($"a", $"b", $"c", $"d").collect())
|
df.select($"a", $"b", $"c", $"d").collect())
|
||||||
|
|
Loading…
Reference in a new issue