[SPARK-4425][SQL] Handle NaN or Infinity cast to Timestamp correctly.
`Cast` from `NaN` or `Infinity` of `Double` or `Float` to `TimestampType` throws `NumberFormatException`. Author: Takuya UESHIN <ueshin@happy-camper.st> Closes #3283 from ueshin/issues/SPARK-4425 and squashes the following commits: 14def0c [Takuya UESHIN] Fix Cast to be able to handle NaN or Infinity to TimestampType.
This commit is contained in:
parent
3a81a1c9e0
commit
566c791931
|
@ -32,6 +32,8 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
|
||||||
override def nullable = (child.dataType, dataType) match {
|
override def nullable = (child.dataType, dataType) match {
|
||||||
case (StringType, _: NumericType) => true
|
case (StringType, _: NumericType) => true
|
||||||
case (StringType, TimestampType) => true
|
case (StringType, TimestampType) => true
|
||||||
|
case (DoubleType, TimestampType) => true
|
||||||
|
case (FloatType, TimestampType) => true
|
||||||
case (StringType, DateType) => true
|
case (StringType, DateType) => true
|
||||||
case (_: NumericType, DateType) => true
|
case (_: NumericType, DateType) => true
|
||||||
case (BooleanType, DateType) => true
|
case (BooleanType, DateType) => true
|
||||||
|
@ -117,10 +119,18 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
|
||||||
buildCast[Decimal](_, d => decimalToTimestamp(d))
|
buildCast[Decimal](_, d => decimalToTimestamp(d))
|
||||||
// TimestampWritable.doubleToTimestamp
|
// TimestampWritable.doubleToTimestamp
|
||||||
case DoubleType =>
|
case DoubleType =>
|
||||||
buildCast[Double](_, d => decimalToTimestamp(Decimal(d)))
|
buildCast[Double](_, d => try {
|
||||||
|
decimalToTimestamp(Decimal(d))
|
||||||
|
} catch {
|
||||||
|
case _: NumberFormatException => null
|
||||||
|
})
|
||||||
// TimestampWritable.floatToTimestamp
|
// TimestampWritable.floatToTimestamp
|
||||||
case FloatType =>
|
case FloatType =>
|
||||||
buildCast[Float](_, f => decimalToTimestamp(Decimal(f)))
|
buildCast[Float](_, f => try {
|
||||||
|
decimalToTimestamp(Decimal(f))
|
||||||
|
} catch {
|
||||||
|
case _: NumberFormatException => null
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
private[this] def decimalToTimestamp(d: Decimal) = {
|
private[this] def decimalToTimestamp(d: Decimal) = {
|
||||||
|
|
|
@ -450,6 +450,11 @@ class ExpressionEvaluationSuite extends FunSuite {
|
||||||
|
|
||||||
// A test for higher precision than millis
|
// A test for higher precision than millis
|
||||||
checkEvaluation(Cast(Cast(0.00000001, TimestampType), DoubleType), 0.00000001)
|
checkEvaluation(Cast(Cast(0.00000001, TimestampType), DoubleType), 0.00000001)
|
||||||
|
|
||||||
|
checkEvaluation(Cast(Literal(Double.NaN), TimestampType), null)
|
||||||
|
checkEvaluation(Cast(Literal(1.0 / 0.0), TimestampType), null)
|
||||||
|
checkEvaluation(Cast(Literal(Float.NaN), TimestampType), null)
|
||||||
|
checkEvaluation(Cast(Literal(1.0f / 0.0f), TimestampType), null)
|
||||||
}
|
}
|
||||||
|
|
||||||
test("null checking") {
|
test("null checking") {
|
||||||
|
|
Loading…
Reference in a new issue