[SPARK-35679][SQL] instantToMicros overflow
### Why are the changes needed? With Long.minValue cast to an instant, secs will be floored in function microsToInstant and cause overflow when multiply with Micros_per_second ``` def microsToInstant(micros: Long): Instant = { val secs = Math.floorDiv(micros, MICROS_PER_SECOND) // Unfolded Math.floorMod(us, MICROS_PER_SECOND) to reuse the result of // the above calculation of `secs` via `floorDiv`. val mos = micros - secs * MICROS_PER_SECOND <- it will overflow here Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS) } ``` But the overflow is acceptable because it won't produce any change to the result However, when convert the instant back to micro value, it will raise Overflow Error ``` def instantToMicros(instant: Instant): Long = { val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) <- It overflow here val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano)) result } ``` Code to reproduce this error ``` instantToMicros(microToInstant(Long.MinValue)) ``` ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Test added Closes #32839 from dgd-contributor/SPARK-35679_instantToMicro. Authored-by: dgd-contributor <dgd_contributor@viettel.com.vn> Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
8dde20a993
commit
aa3de40773
|
@ -378,6 +378,9 @@ object DateTimeUtils {
|
||||||
throw QueryExecutionErrors.cannotCastUTF8StringToDataTypeError(s, TimestampType)
|
throw QueryExecutionErrors.cannotCastUTF8StringToDataTypeError(s, TimestampType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// See issue SPARK-35679
|
||||||
|
// min second cause overflow in instant to micro
|
||||||
|
private val MIN_SECONDS = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given
|
* Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given
|
||||||
|
@ -385,9 +388,14 @@ object DateTimeUtils {
|
||||||
* microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
|
* microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
|
||||||
*/
|
*/
|
||||||
def instantToMicros(instant: Instant): Long = {
|
def instantToMicros(instant: Instant): Long = {
|
||||||
val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND)
|
val secs = instant.getEpochSecond
|
||||||
val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
|
if (secs == MIN_SECONDS) {
|
||||||
result
|
val us = Math.multiplyExact(secs + 1, MICROS_PER_SECOND)
|
||||||
|
Math.addExact(us, NANOSECONDS.toMicros(instant.getNano) - MICROS_PER_SECOND)
|
||||||
|
} else {
|
||||||
|
val us = Math.multiplyExact(secs, MICROS_PER_SECOND)
|
||||||
|
Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -820,4 +820,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test("SPARK-35679: instantToMicros should be able to return microseconds of Long.MinValue") {
|
||||||
|
assert(instantToMicros(microsToInstant(Long.MaxValue)) === Long.MaxValue)
|
||||||
|
assert(instantToMicros(microsToInstant(Long.MinValue)) === Long.MinValue)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue