[SPARK-29187][SQL] Return null from date_part()
for the null field
### What changes were proposed in this pull request? In the PR, I propose to change behavior of the `date_part()` function in handling `null` field, and make it the same as PostgreSQL has. If `field` parameter is `null`, the function should return `null` of the `double` type as PostgreSQL does: ```sql # select date_part(null, date '2019-09-20'); date_part ----------- (1 row) # select pg_typeof(date_part(null, date '2019-09-20')); pg_typeof ------------------ double precision (1 row) ``` ### Why are the changes needed? The `date_part()` function was added to maintain feature parity with PostgreSQL but current behavior of the function is different in handling null as `field`. ### Does this PR introduce any user-facing change? Yes. Before: ```sql spark-sql> select date_part(null, date'2019-09-20'); Error in query: null; line 1 pos 7 ``` After: ```sql spark-sql> select date_part(null, date'2019-09-20'); NULL ``` ### How was this patch tested? Add new tests to `DateFunctionsSuite for 2 cases: - `field` = `null`, `source` = a date literal - `field` = `null`, `source` = a date column Closes #25865 from MaxGekk/date_part-null. Authored-by: Maxim Gekk <max.gekk@gmail.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
parent
ff3a737c75
commit
252b6cf3c9
|
@ -2053,10 +2053,15 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
|
|||
if (!field.foldable) {
|
||||
throw new AnalysisException("The field parameter needs to be a foldable string value.")
|
||||
}
|
||||
val fieldStr = field.eval().asInstanceOf[UTF8String].toString
|
||||
DatePart.parseExtractField(fieldStr, source, {
|
||||
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
|
||||
})
|
||||
val fieldEval = field.eval()
|
||||
if (fieldEval == null) {
|
||||
Literal(null, DoubleType)
|
||||
} else {
|
||||
val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
|
||||
DatePart.parseExtractField(fieldStr, source, {
|
||||
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -66,3 +66,5 @@ select date_part('secs', c) from t;
|
|||
select date_part('not_supported', c) from t;
|
||||
|
||||
select date_part(c, c) from t;
|
||||
|
||||
select date_part(null, c) from t;
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
-- Automatically generated by SQLQueryTestSuite
|
||||
-- Number of queries: 51
|
||||
-- Number of queries: 52
|
||||
|
||||
|
||||
-- !query 0
|
||||
|
@ -410,3 +410,11 @@ struct<>
|
|||
-- !query 50 output
|
||||
org.apache.spark.sql.AnalysisException
|
||||
The field parameter needs to be a foldable string value.;; line 1 pos 7
|
||||
|
||||
|
||||
-- !query 51
|
||||
select date_part(null, c) from t
|
||||
-- !query 51 schema
|
||||
struct<date_part(NULL, t.`c`):double>
|
||||
-- !query 51 output
|
||||
NULL
|
||||
|
|
|
@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
|
|||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.internal.SQLConf
|
||||
import org.apache.spark.sql.test.SharedSparkSession
|
||||
import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
|
||||
import org.apache.spark.unsafe.types.CalendarInterval
|
||||
|
||||
class DateFunctionsSuite extends QueryTest with SharedSparkSession {
|
||||
|
@ -796,4 +797,13 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
|
|||
Seq(Row(Instant.parse(timestamp))))
|
||||
}
|
||||
}
|
||||
|
||||
test("handling null field by date_part") {
|
||||
val input = Seq(Date.valueOf("2019-09-20")).toDF("d")
|
||||
Seq("date_part(null, d)", "date_part(null, date'2019-09-20')").foreach { expr =>
|
||||
val df = input.selectExpr(expr)
|
||||
assert(df.schema.headOption.get.dataType == DoubleType)
|
||||
checkAnswer(df, Row(null))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue