[SPARK-36807][SQL] Merge ANSI interval types to a tightest common type
### What changes were proposed in this pull request?
In the PR, I propose to modify `StructType` to support merging of ANSI interval types with different fields.
### Why are the changes needed?
This will allow merging of schemas from different datasource files.
### Does this PR introduce _any_ user-facing change?
No, the ANSI interval types haven't released yet.
### How was this patch tested?
Added new test to `StructTypeSuite`.
Closes #34049 from MaxGekk/merge-ansi-interval-types.
Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
(cherry picked from commit d2340f8e1c
)
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
3d47c692d2
commit
7fa88b28a5
|
@ -653,6 +653,12 @@ object StructType extends AbstractDataType {
|
|||
case (leftUdt: UserDefinedType[_], rightUdt: UserDefinedType[_])
|
||||
if leftUdt.userClass == rightUdt.userClass => leftUdt
|
||||
|
||||
case (YearMonthIntervalType(lstart, lend), YearMonthIntervalType(rstart, rend)) =>
|
||||
YearMonthIntervalType(Math.min(lstart, rstart).toByte, Math.max(lend, rend).toByte)
|
||||
|
||||
case (DayTimeIntervalType(lstart, lend), DayTimeIntervalType(rstart, rend)) =>
|
||||
DayTimeIntervalType(Math.min(lstart, rstart).toByte, Math.max(lend, rend).toByte)
|
||||
|
||||
case (leftType, rightType) if leftType == rightType =>
|
||||
leftType
|
||||
|
||||
|
|
|
@ -25,7 +25,9 @@ import org.apache.spark.sql.catalyst.plans.SQLHelper
|
|||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.types.{DayTimeIntervalType => DT}
|
||||
import org.apache.spark.sql.types.{YearMonthIntervalType => YM}
|
||||
import org.apache.spark.sql.types.DayTimeIntervalType._
|
||||
import org.apache.spark.sql.types.StructType.fromDDL
|
||||
import org.apache.spark.sql.types.YearMonthIntervalType._
|
||||
|
||||
class StructTypeSuite extends SparkFunSuite with SQLHelper {
|
||||
|
||||
|
@ -382,4 +384,25 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
|
|||
assert(e.getMessage.contains(
|
||||
"Field name a2.element.C.name is invalid: a2.element.c is not a struct"))
|
||||
}
|
||||
|
||||
test("SPARK-36807: Merge ANSI interval types to a tightest common type") {
|
||||
Seq(
|
||||
(YM(YEAR), YM(YEAR)) -> YM(YEAR),
|
||||
(YM(YEAR), YM(MONTH)) -> YM(YEAR, MONTH),
|
||||
(YM(MONTH), YM(MONTH)) -> YM(MONTH),
|
||||
(YM(YEAR, MONTH), YM(YEAR)) -> YM(YEAR, MONTH),
|
||||
(YM(YEAR, MONTH), YM(YEAR, MONTH)) -> YM(YEAR, MONTH),
|
||||
(DT(DAY), DT(DAY)) -> DT(DAY),
|
||||
(DT(SECOND), DT(SECOND)) -> DT(SECOND),
|
||||
(DT(DAY), DT(SECOND)) -> DT(DAY, SECOND),
|
||||
(DT(HOUR, SECOND), DT(DAY, MINUTE)) -> DT(DAY, SECOND),
|
||||
(DT(HOUR, MINUTE), DT(DAY, SECOND)) -> DT(DAY, SECOND)
|
||||
).foreach { case ((i1, i2), expected) =>
|
||||
val st1 = new StructType().add("interval", i1)
|
||||
val st2 = new StructType().add("interval", i2)
|
||||
val expectedStruct = new StructType().add("interval", expected)
|
||||
assert(st1.merge(st2) === expectedStruct)
|
||||
assert(st2.merge(st1) === expectedStruct)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue