ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-36807][SQL] Merge ANSI interval types to a tightest common type

Browse source 
### What changes were proposed in this pull request?
In the PR, I propose to modify `StructType` to support merging of ANSI interval types with different fields.

### Why are the changes needed?
This will allow merging of schemas from different datasource files.

### Does this PR introduce _any_ user-facing change?
No, the ANSI interval types haven't released yet.

### How was this patch tested?
Added new test to `StructTypeSuite`.

Closes #34049 from MaxGekk/merge-ansi-interval-types.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Max Gekk 2021-09-21 10:20:16 +03:00
parent cc182fe6f6
commit d2340f8e1c
2 changed files with 29 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
View file

@ -653,6 +653,12 @@ object StructType extends AbstractDataType {
case (leftUdt: UserDefinedType[_], rightUdt: UserDefinedType[_])
if leftUdt.userClass == rightUdt.userClass => leftUdt
case (YearMonthIntervalType(lstart, lend), YearMonthIntervalType(rstart, rend)) =>
YearMonthIntervalType(Math.min(lstart, rstart).toByte, Math.max(lend, rend).toByte)
case (DayTimeIntervalType(lstart, lend), DayTimeIntervalType(rstart, rend)) =>
DayTimeIntervalType(Math.min(lstart, rstart).toByte, Math.max(lend, rend).toByte)
case (leftType, rightType) if leftType == rightType =>
leftType

23
sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
View file

@ -25,7 +25,9 @@ import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DayTimeIntervalType => DT}
import org.apache.spark.sql.types.{YearMonthIntervalType => YM}
import org.apache.spark.sql.types.DayTimeIntervalType._
import org.apache.spark.sql.types.StructType.fromDDL
import org.apache.spark.sql.types.YearMonthIntervalType._
class StructTypeSuite extends SparkFunSuite with SQLHelper {
@ -382,4 +384,25 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
assert(e.getMessage.contains(
"Field name a2.element.C.name is invalid: a2.element.c is not a struct"))
}
test("SPARK-36807: Merge ANSI interval types to a tightest common type") {
Seq(
(YM(YEAR), YM(YEAR)) -> YM(YEAR),
(YM(YEAR), YM(MONTH)) -> YM(YEAR, MONTH),
(YM(MONTH), YM(MONTH)) -> YM(MONTH),
(YM(YEAR, MONTH), YM(YEAR)) -> YM(YEAR, MONTH),
(YM(YEAR, MONTH), YM(YEAR, MONTH)) -> YM(YEAR, MONTH),
(DT(DAY), DT(DAY)) -> DT(DAY),
(DT(SECOND), DT(SECOND)) -> DT(SECOND),
(DT(DAY), DT(SECOND)) -> DT(DAY, SECOND),
(DT(HOUR, SECOND), DT(DAY, MINUTE)) -> DT(DAY, SECOND),
(DT(HOUR, MINUTE), DT(DAY, SECOND)) -> DT(DAY, SECOND)
).foreach { case ((i1, i2), expected) =>
val st1 = new StructType().add("interval", i1)
val st2 = new StructType().add("interval", i2)
val expectedStruct = new StructType().add("interval", expected)
assert(st1.merge(st2) === expectedStruct)
assert(st2.merge(st1) === expectedStruct)
}
}
}