ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-28308][CORE] CalendarInterval sub-second part should be padded before parsing

Browse source 
## What changes were proposed in this pull request?

The sub-second part of the interval should be padded before parsing. Currently, Spark gives a correct value only when there is 9 digits below `.`.
```
spark-sql> select interval '0 0:0:0.123456789' day to second;
interval 123 milliseconds 456 microseconds

spark-sql> select interval '0 0:0:0.12345678' day to second;
interval 12 milliseconds 345 microseconds

spark-sql> select interval '0 0:0:0.1234' day to second;
interval 1 microseconds
```

## How was this patch tested?

Pass the Jenkins with the fixed test cases.

Closes #25079 from dongjoon-hyun/SPARK-28308.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
Dongjoon Hyun 2019-07-08 19:40:41 -07:00
parent 28ea445c43
commit a5ff9221fc
3 changed files with 9 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
View file

@ -178,7 +178,8 @@ public final class CalendarInterval implements Serializable {
long minutes = toLongWithRange("minute", m.group(5), 0, 59);
long seconds = toLongWithRange("second", m.group(6), 0, 59);
// Hive allow nanosecond precision interval
long nanos = toLongWithRange("nanosecond", m.group(8), 0L, 999999999L);
String nanoStr = m.group(8) == null ? null : (m.group(8) + "000000000").substring(0, 9);
long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
result = new CalendarInterval(0, sign * (
days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE +
seconds * MICROS_PER_SECOND + nanos / 1000L));

3
common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
View file

@ -162,7 +162,8 @@ public class CalendarIntervalSuite {
assertEquals(fromDayTimeString(input), i);
input = "10 0:12:0.888";
i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE);
i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
888 * MICROS_PER_MILLI);
assertEquals(fromDayTimeString(input), i);
input = "-3 0:0:0";

8
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
View file

@ -1179,14 +1179,16 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
test("Convert hive interval term into Literal of CalendarIntervalType") {
checkAnswer(sql("select interval '0 0:0:0.1' day to second"),
Row(CalendarInterval.fromString("interval 100 milliseconds")))
checkAnswer(sql("select interval '10-9' year to month"),
Row(CalendarInterval.fromString("interval 10 years 9 months")))
checkAnswer(sql("select interval '20 15:40:32.99899999' day to second"),
Row(CalendarInterval.fromString("interval 2 weeks 6 days 15 hours 40 minutes " +
"32 seconds 99 milliseconds 899 microseconds")))
"32 seconds 998 milliseconds 999 microseconds")))
checkAnswer(sql("select interval '15:40:32.99899999' hour to second"),
Row(CalendarInterval.fromString("interval 15 hours 40 minutes 32 seconds 99 milliseconds " +
"899 microseconds")))
Row(CalendarInterval.fromString("interval 15 hours 40 minutes 32 seconds 998 milliseconds " +
"999 microseconds")))
checkAnswer(sql("select interval '30' year"),
Row(CalendarInterval.fromString("interval 30 years")))
checkAnswer(sql("select interval '25' month"),