[SPARK-31166][SQL] UNION map<null, null> and other maps should not fail
### What changes were proposed in this pull request? After https://github.com/apache/spark/pull/27542, `map()` returns `map<null, null>` instead of `map<string, string>`. However, this breaks queries which union `map()` and other maps. The reason is, `TypeCoercion` rules and `Cast` think it's illegal to cast null type map key to other types, as it makes the key nullable, but it's actually legal. This PR fixes it. ### Why are the changes needed? To avoid breaking queries. ### Does this PR introduce any user-facing change? Yes, now some queries that work in 2.x can work in 3.0 as well. ### How was this patch tested? new test Closes #27926 from cloud-fan/bug. Authored-by: Wenchen Fan <wenchen@databricks.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
93088f79cc
commit
d7b97a1d0d
|
@ -158,7 +158,7 @@ object TypeCoercion {
|
|||
}
|
||||
case (MapType(kt1, vt1, valueContainsNull1), MapType(kt2, vt2, valueContainsNull2)) =>
|
||||
findTypeFunc(kt1, kt2)
|
||||
.filter { kt => !Cast.forceNullable(kt1, kt) && !Cast.forceNullable(kt2, kt) }
|
||||
.filter { kt => Cast.canCastMapKeyNullSafe(kt1, kt) && Cast.canCastMapKeyNullSafe(kt2, kt) }
|
||||
.flatMap { kt =>
|
||||
findTypeFunc(vt1, vt2).map { vt =>
|
||||
MapType(kt, vt, valueContainsNull1 || valueContainsNull2 ||
|
||||
|
|
|
@ -77,8 +77,7 @@ object Cast {
|
|||
resolvableNullability(fn || forceNullable(fromType, toType), tn)
|
||||
|
||||
case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
|
||||
canCast(fromKey, toKey) &&
|
||||
(!forceNullable(fromKey, toKey)) &&
|
||||
canCast(fromKey, toKey) && canCastMapKeyNullSafe(fromKey, toKey) &&
|
||||
canCast(fromValue, toValue) &&
|
||||
resolvableNullability(fn || forceNullable(fromValue, toValue), tn)
|
||||
|
||||
|
@ -98,6 +97,11 @@ object Cast {
|
|||
case _ => false
|
||||
}
|
||||
|
||||
def canCastMapKeyNullSafe(fromType: DataType, toType: DataType): Boolean = {
|
||||
// If the original map key type is NullType, it's OK as the map must be empty.
|
||||
fromType == NullType || !forceNullable(fromType, toType)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true if we need to use the `timeZone` information casting `from` type to `to` type.
|
||||
* The patterns matched reflect the current implementation in the Cast node.
|
||||
|
|
|
@ -3487,6 +3487,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
test("SPARK-31166: UNION map<null, null> and other maps should not fail") {
|
||||
checkAnswer(
|
||||
sql("(SELECT map()) UNION ALL (SELECT map(1, 2))"),
|
||||
Seq(Row(Map[Int, Int]()), Row(Map(1 -> 2))))
|
||||
}
|
||||
}
|
||||
|
||||
case class Foo(bar: Option[String])
|
||||
|
|
Loading…
Reference in a new issue