Revert "[SPARK-29462] The data type of "array()" should be array<null>"
This reverts commit 0dcd739534
.
This commit is contained in:
parent
eb79af8dae
commit
80fbc382a6
|
@ -217,8 +217,6 @@ license: |
|
||||||
For example `SELECT timestamp 'tomorrow';`.
|
For example `SELECT timestamp 'tomorrow';`.
|
||||||
|
|
||||||
- Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
|
- Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
|
||||||
|
|
||||||
- Since Spark 3.0, when `array` function is called without parameters, it returns an empty array with `NullType` data type. In Spark version 2.4 and earlier, the data type of the result is `StringType`.
|
|
||||||
|
|
||||||
- Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
|
- Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
|
||||||
|
|
||||||
|
|
|
@ -47,7 +47,7 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
|
||||||
override def dataType: ArrayType = {
|
override def dataType: ArrayType = {
|
||||||
ArrayType(
|
ArrayType(
|
||||||
TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(children.map(_.dataType))
|
TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(children.map(_.dataType))
|
||||||
.getOrElse(NullType),
|
.getOrElse(StringType),
|
||||||
containsNull = children.exists(_.nullable))
|
containsNull = children.exists(_.nullable))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3400,9 +3400,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
|
||||||
).foreach(assertValuesDoNotChangeAfterCoalesceOrUnion(_))
|
).foreach(assertValuesDoNotChangeAfterCoalesceOrUnion(_))
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-21281 use string types by default if map have no argument") {
|
test("SPARK-21281 use string types by default if array and map have no argument") {
|
||||||
val ds = spark.range(1)
|
val ds = spark.range(1)
|
||||||
var expectedSchema = new StructType()
|
var expectedSchema = new StructType()
|
||||||
|
.add("x", ArrayType(StringType, containsNull = false), nullable = false)
|
||||||
|
assert(ds.select(array().as("x")).schema == expectedSchema)
|
||||||
|
expectedSchema = new StructType()
|
||||||
.add("x", MapType(StringType, StringType, valueContainsNull = false), nullable = false)
|
.add("x", MapType(StringType, StringType, valueContainsNull = false), nullable = false)
|
||||||
assert(ds.select(map().as("x")).schema == expectedSchema)
|
assert(ds.select(map().as("x")).schema == expectedSchema)
|
||||||
}
|
}
|
||||||
|
@ -3460,13 +3463,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
|
||||||
checkAnswer(df.select("x").filter("exists(i, x -> x % d == 0)"),
|
checkAnswer(df.select("x").filter("exists(i, x -> x % d == 0)"),
|
||||||
Seq(Row(1)))
|
Seq(Row(1)))
|
||||||
}
|
}
|
||||||
|
|
||||||
test("SPARK-29462: Use null type by default if array have no argument") {
|
|
||||||
val ds = spark.range(1)
|
|
||||||
var expectedSchema = new StructType()
|
|
||||||
.add("x", ArrayType(NullType, containsNull = false), nullable = false)
|
|
||||||
assert(ds.select(array().as("x")).schema == expectedSchema)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
object DataFrameFunctionsSuite {
|
object DataFrameFunctionsSuite {
|
||||||
|
|
Loading…
Reference in a new issue