[SPARK-35679][SQL] instantToMicros overflow
### Why are the changes needed? With Long.minValue cast to an instant, secs will be floored in function microsToInstant and cause overflow when multiply with Micros_per_second ``` def microsToInstant(micros: Long): Instant = { val secs = Math.floorDiv(micros, MICROS_PER_SECOND) // Unfolded Math.floorMod(us, MICROS_PER_SECOND) to reuse the result of // the above calculation of `secs` via `floorDiv`. val mos = micros - secs * MICROS_PER_SECOND <- it will overflow here Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS) } ``` But the overflow is acceptable because it won't produce any change to the result However, when convert the instant back to micro value, it will raise Overflow Error ``` def instantToMicros(instant: Instant): Long = { val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) <- It overflow here val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) result } ``` Code to reproduce this error ``` instantToMicros(microToInstant(Long.MinValue)) ``` ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Test added Closes #32839 from dgd-contributor/SPARK-35679_instantToMicro. Authored-by: dgd-contributor <dgd_contributor@viettel.com.vn> Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
8dde20a993
commit
aa3de40773
|
@ -378,6 +378,9 @@ object DateTimeUtils {
|
|||
throw QueryExecutionErrors.cannotCastUTF8StringToDataTypeError(s, TimestampType)
|
||||
}
|
||||
}
|
||||
// See issue SPARK-35679
|
||||
// min second cause overflow in instant to micro
|
||||
private val MIN_SECONDS = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
|
||||
|
||||
/**
|
||||
* Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given
|
||||
|
@ -385,9 +388,14 @@ object DateTimeUtils {
|
|||
* microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
|
||||
*/
|
||||
def instantToMicros(instant: Instant): Long = {
|
||||
val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND)
|
||||
val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
|
||||
result
|
||||
val secs = instant.getEpochSecond
|
||||
if (secs == MIN_SECONDS) {
|
||||
val us = Math.multiplyExact(secs + 1, MICROS_PER_SECOND)
|
||||
Math.addExact(us, NANOSECONDS.toMicros(instant.getNano) - MICROS_PER_SECOND)
|
||||
} else {
|
||||
val us = Math.multiplyExact(secs, MICROS_PER_SECOND)
|
||||
Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -820,4 +820,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
test("SPARK-35679: instantToMicros should be able to return microseconds of Long.MinValue") {
|
||||
assert(instantToMicros(microsToInstant(Long.MaxValue)) === Long.MaxValue)
|
||||
assert(instantToMicros(microsToInstant(Long.MinValue)) === Long.MinValue)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue