[SPARK-29187][SQL] Return null from date_part() for the null field

### What changes were proposed in this pull request?

In the PR, I propose to change behavior of the `date_part()` function in handling `null` field, and make it the same as PostgreSQL has. If `field` parameter is `null`, the function should return `null` of the `double` type as PostgreSQL does:
```sql
# select date_part(null, date '2019-09-20');
 date_part
-----------

(1 row)

# select pg_typeof(date_part(null, date '2019-09-20'));
    pg_typeof
------------------
 double precision
(1 row)
```

### Why are the changes needed?
The `date_part()` function was added to maintain feature parity with PostgreSQL but current behavior of the function is different in handling null as `field`.

### Does this PR introduce any user-facing change?
Yes.

Before:
```sql
spark-sql> select date_part(null, date'2019-09-20');
Error in query: null; line 1 pos 7
```

After:
```sql
spark-sql> select date_part(null, date'2019-09-20');
NULL
```

### How was this patch tested?
Add new tests to `DateFunctionsSuite for 2 cases:
- `field` = `null`, `source` = a date literal
- `field` = `null`, `source` = a date column

Closes #25865 from MaxGekk/date_part-null.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
Maxim Gekk 2019-09-20 20:28:56 -07:00 committed by Dongjoon Hyun
parent ff3a737c75
commit 252b6cf3c9
4 changed files with 30 additions and 5 deletions

View file

@ -2053,10 +2053,15 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
if (!field.foldable) {
throw new AnalysisException("The field parameter needs to be a foldable string value.")
}
val fieldStr = field.eval().asInstanceOf[UTF8String].toString
DatePart.parseExtractField(fieldStr, source, {
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
})
val fieldEval = field.eval()
if (fieldEval == null) {
Literal(null, DoubleType)
} else {
val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
DatePart.parseExtractField(fieldStr, source, {
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
})
}
})
}

View file

@ -66,3 +66,5 @@ select date_part('secs', c) from t;
select date_part('not_supported', c) from t;
select date_part(c, c) from t;
select date_part(null, c) from t;

View file

@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 51
-- Number of queries: 52
-- !query 0
@ -410,3 +410,11 @@ struct<>
-- !query 50 output
org.apache.spark.sql.AnalysisException
The field parameter needs to be a foldable string value.;; line 1 pos 7
-- !query 51
select date_part(null, c) from t
-- !query 51 schema
struct<date_part(NULL, t.`c`):double>
-- !query 51 output
NULL

View file

@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
import org.apache.spark.unsafe.types.CalendarInterval
class DateFunctionsSuite extends QueryTest with SharedSparkSession {
@ -796,4 +797,13 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
Seq(Row(Instant.parse(timestamp))))
}
}
test("handling null field by date_part") {
val input = Seq(Date.valueOf("2019-09-20")).toDF("d")
Seq("date_part(null, d)", "date_part(null, date'2019-09-20')").foreach { expr =>
val df = input.selectExpr(expr)
assert(df.schema.headOption.get.dataType == DoubleType)
checkAnswer(df, Row(null))
}
}
}