[SPARK-29607][SQL] Move static methods from CalendarInterval to IntervalUtils
### What changes were proposed in this pull request? In the PR, I propose to move all static methods from the `CalendarInterval` class to the `IntervalUtils` object. All those methods are rewritten from Java to Scala. ### Why are the changes needed? - For consistency with other helper methods. Such methods were placed to the helper object `IntervalUtils`, see https://github.com/apache/spark/pull/26190 - Taking into account that `CalendarInterval` will be fully exposed to users in the future (see https://github.com/apache/spark/pull/25022), it would be nice to clean it up by moving service methods to an internal object. ### Does this PR introduce any user-facing change? No ### How was this patch tested? - By moved tests from `CalendarIntervalSuite` to `IntervalUtilsSuite` - By existing test suites Closes #26261 from MaxGekk/refactoring-calendar-interval. Authored-by: Maxim Gekk <max.gekk@gmail.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
Maxim Gekk
Wenchen Fan
parent
37690dea10
commit
44c1c03924
|
|
@ -18,8 +18,6 @@
|
|||
package org.apache.spark.unsafe.types;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* The internal representation of interval type.
|
||||
|
|
@ -32,209 +30,6 @@ public final class CalendarInterval implements Serializable {
|
|||
public static final long MICROS_PER_DAY = MICROS_PER_HOUR * 24;
|
||||
public static final long MICROS_PER_WEEK = MICROS_PER_DAY * 7;
|
||||
|
||||
private static Pattern yearMonthPattern = Pattern.compile(
|
||||
"^([+|-])?(\\d+)-(\\d+)$");
|
||||
|
||||
private static Pattern dayTimePattern = Pattern.compile(
|
||||
"^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$");
|
||||
|
||||
public static long toLongWithRange(String fieldName,
|
||||
String s, long minValue, long maxValue) throws IllegalArgumentException {
|
||||
long result = 0;
|
||||
if (s != null) {
|
||||
result = Long.parseLong(s);
|
||||
if (result < minValue || result > maxValue) {
|
||||
throw new IllegalArgumentException(String.format("%s %d outside range [%d, %d]",
|
||||
fieldName, result, minValue, maxValue));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse YearMonth string in form: [-]YYYY-MM
|
||||
*
|
||||
* adapted from HiveIntervalYearMonth.valueOf
|
||||
*/
|
||||
public static CalendarInterval fromYearMonthString(String s) throws IllegalArgumentException {
|
||||
CalendarInterval result = null;
|
||||
if (s == null) {
|
||||
throw new IllegalArgumentException("Interval year-month string was null");
|
||||
}
|
||||
s = s.trim();
|
||||
Matcher m = yearMonthPattern.matcher(s);
|
||||
if (!m.matches()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Interval string does not match year-month format of 'y-m': " + s);
|
||||
} else {
|
||||
try {
|
||||
int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
|
||||
int years = (int) toLongWithRange("year", m.group(2), 0, Integer.MAX_VALUE);
|
||||
int months = (int) toLongWithRange("month", m.group(3), 0, 11);
|
||||
result = new CalendarInterval(sign * (years * 12 + months), 0);
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error parsing interval year-month string: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn
|
||||
*
|
||||
* adapted from HiveIntervalDayTime.valueOf
|
||||
*/
|
||||
public static CalendarInterval fromDayTimeString(String s) throws IllegalArgumentException {
|
||||
return fromDayTimeString(s, "day", "second");
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn
|
||||
*
|
||||
* adapted from HiveIntervalDayTime.valueOf.
|
||||
* Below interval conversion patterns are supported:
|
||||
* - DAY TO (HOUR|MINUTE|SECOND)
|
||||
* - HOUR TO (MINUTE|SECOND)
|
||||
* - MINUTE TO SECOND
|
||||
*/
|
||||
public static CalendarInterval fromDayTimeString(String s, String from, String to)
|
||||
throws IllegalArgumentException {
|
||||
CalendarInterval result = null;
|
||||
if (s == null) {
|
||||
throw new IllegalArgumentException("Interval day-time string was null");
|
||||
}
|
||||
s = s.trim();
|
||||
Matcher m = dayTimePattern.matcher(s);
|
||||
if (!m.matches()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Interval string does not match day-time format of 'd h:m:s.n': " + s);
|
||||
} else {
|
||||
try {
|
||||
int sign = m.group(1) != null && m.group(1).equals("-") ? -1 : 1;
|
||||
long days = m.group(2) == null ? 0 : toLongWithRange("day", m.group(3),
|
||||
0, Integer.MAX_VALUE);
|
||||
long hours = 0;
|
||||
long minutes;
|
||||
long seconds = 0;
|
||||
if (m.group(5) != null || from.equals("minute")) { // 'HH:mm:ss' or 'mm:ss minute'
|
||||
hours = toLongWithRange("hour", m.group(5), 0, 23);
|
||||
minutes = toLongWithRange("minute", m.group(6), 0, 59);
|
||||
seconds = toLongWithRange("second", m.group(7), 0, 59);
|
||||
} else if (m.group(8) != null){ // 'mm:ss.nn'
|
||||
minutes = toLongWithRange("minute", m.group(6), 0, 59);
|
||||
seconds = toLongWithRange("second", m.group(7), 0, 59);
|
||||
} else { // 'HH:mm'
|
||||
hours = toLongWithRange("hour", m.group(6), 0, 23);
|
||||
minutes = toLongWithRange("second", m.group(7), 0, 59);
|
||||
}
|
||||
// Hive allow nanosecond precision interval
|
||||
String nanoStr = m.group(9) == null ? null : (m.group(9) + "000000000").substring(0, 9);
|
||||
long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
|
||||
switch (to) {
|
||||
case "hour":
|
||||
minutes = 0;
|
||||
seconds = 0;
|
||||
nanos = 0;
|
||||
break;
|
||||
case "minute":
|
||||
seconds = 0;
|
||||
nanos = 0;
|
||||
break;
|
||||
case "second":
|
||||
// No-op
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
String.format("Cannot support (interval '%s' %s to %s) expression", s, from, to));
|
||||
}
|
||||
result = new CalendarInterval(0, sign * (
|
||||
days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE +
|
||||
seconds * MICROS_PER_SECOND + nanos / 1000L));
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error parsing interval day-time string: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static CalendarInterval fromUnitStrings(String[] units, String[] values)
|
||||
throws IllegalArgumentException {
|
||||
assert units.length == values.length;
|
||||
int months = 0;
|
||||
long microseconds = 0;
|
||||
|
||||
for (int i = 0; i < units.length; i++) {
|
||||
try {
|
||||
switch (units[i]) {
|
||||
case "year":
|
||||
months = Math.addExact(months, Math.multiplyExact(Integer.parseInt(values[i]), 12));
|
||||
break;
|
||||
case "month":
|
||||
months = Math.addExact(months, Integer.parseInt(values[i]));
|
||||
break;
|
||||
case "week":
|
||||
microseconds = Math.addExact(
|
||||
microseconds,
|
||||
Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_WEEK));
|
||||
break;
|
||||
case "day":
|
||||
microseconds = Math.addExact(
|
||||
microseconds,
|
||||
Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_DAY));
|
||||
break;
|
||||
case "hour":
|
||||
microseconds = Math.addExact(
|
||||
microseconds,
|
||||
Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_HOUR));
|
||||
break;
|
||||
case "minute":
|
||||
microseconds = Math.addExact(
|
||||
microseconds,
|
||||
Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_MINUTE));
|
||||
break;
|
||||
case "second": {
|
||||
microseconds = Math.addExact(microseconds, parseSecondNano(values[i]));
|
||||
break;
|
||||
}
|
||||
case "millisecond":
|
||||
microseconds = Math.addExact(
|
||||
microseconds,
|
||||
Math.multiplyExact(Long.parseLong(values[i]), MICROS_PER_MILLI));
|
||||
break;
|
||||
case "microsecond":
|
||||
microseconds = Math.addExact(microseconds, Long.parseLong(values[i]));
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException("Error parsing interval string: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return new CalendarInterval(months, microseconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse second_nano string in ss.nnnnnnnnn format to microseconds
|
||||
*/
|
||||
public static long parseSecondNano(String secondNano) throws IllegalArgumentException {
|
||||
String[] parts = secondNano.split("\\.");
|
||||
if (parts.length == 1) {
|
||||
return toLongWithRange("second", parts[0], Long.MIN_VALUE / MICROS_PER_SECOND,
|
||||
Long.MAX_VALUE / MICROS_PER_SECOND) * MICROS_PER_SECOND;
|
||||
|
||||
} else if (parts.length == 2) {
|
||||
long seconds = parts[0].equals("") ? 0L : toLongWithRange("second", parts[0],
|
||||
Long.MIN_VALUE / MICROS_PER_SECOND, Long.MAX_VALUE / MICROS_PER_SECOND);
|
||||
long nanos = toLongWithRange("nanosecond", parts[1], 0L, 999999999L);
|
||||
return seconds * MICROS_PER_SECOND + nanos / 1000L;
|
||||
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
"Interval string does not match second-nano format of ss.nnnnnnnnn");
|
||||
}
|
||||
}
|
||||
|
||||
public final int months;
|
||||
public final long microseconds;
|
||||
|
||||
|
|
|
|||
|
|
@ -60,72 +60,6 @@ public class CalendarIntervalSuite {
|
|||
assertEquals("interval 2 years 10 months 3 weeks 13 hours 123 microseconds", i.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void fromYearMonthStringTest() {
|
||||
String input;
|
||||
CalendarInterval i;
|
||||
|
||||
input = "99-10";
|
||||
i = new CalendarInterval(99 * 12 + 10, 0L);
|
||||
assertEquals(fromYearMonthString(input), i);
|
||||
|
||||
input = "-8-10";
|
||||
i = new CalendarInterval(-8 * 12 - 10, 0L);
|
||||
assertEquals(fromYearMonthString(input), i);
|
||||
|
||||
try {
|
||||
input = "99-15";
|
||||
fromYearMonthString(input);
|
||||
fail("Expected to throw an exception for the invalid input");
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertTrue(e.getMessage().contains("month 15 outside range"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void fromDayTimeStringTest() {
|
||||
String input;
|
||||
CalendarInterval i;
|
||||
|
||||
input = "5 12:40:30.999999999";
|
||||
i = new CalendarInterval(0, 5 * MICROS_PER_DAY + 12 * MICROS_PER_HOUR +
|
||||
40 * MICROS_PER_MINUTE + 30 * MICROS_PER_SECOND + 999999L);
|
||||
assertEquals(fromDayTimeString(input), i);
|
||||
|
||||
input = "10 0:12:0.888";
|
||||
i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
|
||||
888 * MICROS_PER_MILLI);
|
||||
assertEquals(fromDayTimeString(input), i);
|
||||
|
||||
input = "-3 0:0:0";
|
||||
i = new CalendarInterval(0, -3 * MICROS_PER_DAY);
|
||||
assertEquals(fromDayTimeString(input), i);
|
||||
|
||||
try {
|
||||
input = "5 30:12:20";
|
||||
fromDayTimeString(input);
|
||||
fail("Expected to throw an exception for the invalid input");
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertTrue(e.getMessage().contains("hour 30 outside range"));
|
||||
}
|
||||
|
||||
try {
|
||||
input = "5 30-12";
|
||||
fromDayTimeString(input);
|
||||
fail("Expected to throw an exception for the invalid input");
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertTrue(e.getMessage().contains("not match day-time format"));
|
||||
}
|
||||
|
||||
try {
|
||||
input = "5 1:12:20";
|
||||
fromDayTimeString(input, "hour", "microsecond");
|
||||
fail("Expected to throw an exception for the invalid convention type");
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertTrue(e.getMessage().contains("Cannot support (interval"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void addTest() {
|
||||
CalendarInterval input1 = new CalendarInterval(3, 1 * MICROS_PER_HOUR);
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
|
|||
}.toArray
|
||||
val values = ctx.intervalValue().asScala.map(getIntervalValue).toArray
|
||||
try {
|
||||
CalendarInterval.fromUnitStrings(units, values)
|
||||
IntervalUtils.fromUnitStrings(units, values)
|
||||
} catch {
|
||||
case i: IllegalArgumentException =>
|
||||
val e = new ParseException(i.getMessage, ctx)
|
||||
|
|
@ -1953,21 +1953,21 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
|
|||
val unitText = unit.getText.toLowerCase(Locale.ROOT)
|
||||
val interval = (unitText, Option(to).map(_.getText.toLowerCase(Locale.ROOT))) match {
|
||||
case (u, None) =>
|
||||
CalendarInterval.fromUnitStrings(Array(normalizeInternalUnit(u)), Array(s))
|
||||
IntervalUtils.fromUnitStrings(Array(normalizeInternalUnit(u)), Array(s))
|
||||
case ("year", Some("month")) =>
|
||||
CalendarInterval.fromYearMonthString(s)
|
||||
IntervalUtils.fromYearMonthString(s)
|
||||
case ("day", Some("hour")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "day", "hour")
|
||||
IntervalUtils.fromDayTimeString(s, "day", "hour")
|
||||
case ("day", Some("minute")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "day", "minute")
|
||||
IntervalUtils.fromDayTimeString(s, "day", "minute")
|
||||
case ("day", Some("second")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "day", "second")
|
||||
IntervalUtils.fromDayTimeString(s, "day", "second")
|
||||
case ("hour", Some("minute")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "hour", "minute")
|
||||
IntervalUtils.fromDayTimeString(s, "hour", "minute")
|
||||
case ("hour", Some("second")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "hour", "second")
|
||||
IntervalUtils.fromDayTimeString(s, "hour", "second")
|
||||
case ("minute", Some("second")) =>
|
||||
CalendarInterval.fromDayTimeString(s, "minute", "second")
|
||||
IntervalUtils.fromDayTimeString(s, "minute", "second")
|
||||
case (from, Some(t)) =>
|
||||
throw new ParseException(s"Intervals FROM $from TO $t are not supported.", ctx)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@
|
|||
|
||||
package org.apache.spark.sql.catalyst.util
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import scala.util.control.NonFatal
|
||||
|
||||
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
|
||||
import org.apache.spark.sql.types.Decimal
|
||||
import org.apache.spark.unsafe.types.CalendarInterval
|
||||
|
|
@ -119,4 +123,198 @@ object IntervalUtils {
|
|||
case _: IllegalArgumentException => null
|
||||
}
|
||||
}
|
||||
|
||||
private def toLongWithRange(
|
||||
fieldName: String,
|
||||
s: String,
|
||||
minValue: Long,
|
||||
maxValue: Long): Long = {
|
||||
val result = if (s == null) 0L else s.toLong
|
||||
require(minValue <= result && result <= maxValue,
|
||||
s"$fieldName $result outside range [$minValue, $maxValue]")
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
private val yearMonthPattern = "^([+|-])?(\\d+)-(\\d+)$".r
|
||||
|
||||
/**
|
||||
* Parse YearMonth string in form: [+|-]YYYY-MM
|
||||
*
|
||||
* adapted from HiveIntervalYearMonth.valueOf
|
||||
*/
|
||||
def fromYearMonthString(input: String): CalendarInterval = {
|
||||
require(input != null, "Interval year-month string must be not null")
|
||||
def toInterval(yearStr: String, monthStr: String): CalendarInterval = {
|
||||
try {
|
||||
val years = toLongWithRange("year", yearStr, 0, Integer.MAX_VALUE).toInt
|
||||
val months = toLongWithRange("month", monthStr, 0, 11).toInt
|
||||
val totalMonths = Math.addExact(Math.multiplyExact(years, 12), months)
|
||||
new CalendarInterval(totalMonths, 0)
|
||||
} catch {
|
||||
case NonFatal(e) =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Error parsing interval year-month string: ${e.getMessage}", e)
|
||||
}
|
||||
}
|
||||
assert(input.length == input.trim.length)
|
||||
input match {
|
||||
case yearMonthPattern("-", yearStr, monthStr) =>
|
||||
toInterval(yearStr, monthStr).negate()
|
||||
case yearMonthPattern(_, yearStr, monthStr) =>
|
||||
toInterval(yearStr, monthStr)
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Interval string does not match year-month format of 'y-m': $input")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn
|
||||
*
|
||||
* adapted from HiveIntervalDayTime.valueOf
|
||||
*/
|
||||
def fromDayTimeString(s: String): CalendarInterval = {
|
||||
fromDayTimeString(s, "day", "second")
|
||||
}
|
||||
|
||||
private val dayTimePattern =
|
||||
"^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r
|
||||
|
||||
/**
|
||||
* Parse dayTime string in form: [-]d HH:mm:ss.nnnnnnnnn and [-]HH:mm:ss.nnnnnnnnn
|
||||
*
|
||||
* adapted from HiveIntervalDayTime.valueOf.
|
||||
* Below interval conversion patterns are supported:
|
||||
* - DAY TO (HOUR|MINUTE|SECOND)
|
||||
* - HOUR TO (MINUTE|SECOND)
|
||||
* - MINUTE TO SECOND
|
||||
*/
|
||||
def fromDayTimeString(input: String, from: String, to: String): CalendarInterval = {
|
||||
require(input != null, "Interval day-time string must be not null")
|
||||
assert(input.length == input.trim.length)
|
||||
val m = dayTimePattern.pattern.matcher(input)
|
||||
require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input")
|
||||
|
||||
try {
|
||||
val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1
|
||||
val days = if (m.group(2) == null) {
|
||||
0
|
||||
} else {
|
||||
toLongWithRange("day", m.group(3), 0, Integer.MAX_VALUE)
|
||||
}
|
||||
var hours: Long = 0L
|
||||
var minutes: Long = 0L
|
||||
var seconds: Long = 0L
|
||||
if (m.group(5) != null || from == "minute") { // 'HH:mm:ss' or 'mm:ss minute'
|
||||
hours = toLongWithRange("hour", m.group(5), 0, 23)
|
||||
minutes = toLongWithRange("minute", m.group(6), 0, 59)
|
||||
seconds = toLongWithRange("second", m.group(7), 0, 59)
|
||||
} else if (m.group(8) != null) { // 'mm:ss.nn'
|
||||
minutes = toLongWithRange("minute", m.group(6), 0, 59)
|
||||
seconds = toLongWithRange("second", m.group(7), 0, 59)
|
||||
} else { // 'HH:mm'
|
||||
hours = toLongWithRange("hour", m.group(6), 0, 23)
|
||||
minutes = toLongWithRange("second", m.group(7), 0, 59)
|
||||
}
|
||||
// Hive allow nanosecond precision interval
|
||||
val nanoStr = if (m.group(9) == null) {
|
||||
null
|
||||
} else {
|
||||
(m.group(9) + "000000000").substring(0, 9)
|
||||
}
|
||||
var nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L)
|
||||
to match {
|
||||
case "hour" =>
|
||||
minutes = 0
|
||||
seconds = 0
|
||||
nanos = 0
|
||||
case "minute" =>
|
||||
seconds = 0
|
||||
nanos = 0
|
||||
case "second" =>
|
||||
// No-op
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Cannot support (interval '$input' $from to $to) expression")
|
||||
}
|
||||
var micros = nanos / DateTimeUtils.NANOS_PER_MICROS
|
||||
micros = Math.addExact(micros, Math.multiplyExact(days, DateTimeUtils.MICROS_PER_DAY))
|
||||
micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR))
|
||||
micros = Math.addExact(micros, Math.multiplyExact(minutes, MICROS_PER_MINUTE))
|
||||
micros = Math.addExact(micros, Math.multiplyExact(seconds, DateTimeUtils.MICROS_PER_SECOND))
|
||||
new CalendarInterval(0, sign * micros)
|
||||
} catch {
|
||||
case e: Exception =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Error parsing interval day-time string: ${e.getMessage}", e)
|
||||
}
|
||||
}
|
||||
|
||||
def fromUnitStrings(units: Array[String], values: Array[String]): CalendarInterval = {
|
||||
assert(units.length == values.length)
|
||||
var months: Int = 0
|
||||
var microseconds: Long = 0
|
||||
var i = 0
|
||||
while (i < units.length) {
|
||||
try {
|
||||
units(i) match {
|
||||
case "year" =>
|
||||
months = Math.addExact(months, Math.multiplyExact(values(i).toInt, 12))
|
||||
case "month" =>
|
||||
months = Math.addExact(months, values(i).toInt)
|
||||
case "week" =>
|
||||
val weeksUs = Math.multiplyExact(values(i).toLong, 7 * DateTimeUtils.MICROS_PER_DAY)
|
||||
microseconds = Math.addExact(microseconds, weeksUs)
|
||||
case "day" =>
|
||||
val daysUs = Math.multiplyExact(values(i).toLong, DateTimeUtils.MICROS_PER_DAY)
|
||||
microseconds = Math.addExact(microseconds, daysUs)
|
||||
case "hour" =>
|
||||
val hoursUs = Math.multiplyExact(values(i).toLong, MICROS_PER_HOUR)
|
||||
microseconds = Math.addExact(microseconds, hoursUs)
|
||||
case "minute" =>
|
||||
val minutesUs = Math.multiplyExact(values(i).toLong, MICROS_PER_MINUTE)
|
||||
microseconds = Math.addExact(microseconds, minutesUs)
|
||||
case "second" =>
|
||||
microseconds = Math.addExact(microseconds, parseSecondNano(values(i)))
|
||||
case "millisecond" =>
|
||||
val millisUs = Math.multiplyExact(values(i).toLong, DateTimeUtils.MICROS_PER_MILLIS)
|
||||
microseconds = Math.addExact(microseconds, millisUs)
|
||||
case "microsecond" =>
|
||||
microseconds = Math.addExact(microseconds, values(i).toLong)
|
||||
}
|
||||
} catch {
|
||||
case e: Exception =>
|
||||
throw new IllegalArgumentException(s"Error parsing interval string: ${e.getMessage}", e)
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
new CalendarInterval(months, microseconds)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse second_nano string in ss.nnnnnnnnn format to microseconds
|
||||
*/
|
||||
private def parseSecondNano(secondNano: String): Long = {
|
||||
def parseSeconds(secondsStr: String): Long = {
|
||||
toLongWithRange(
|
||||
"second",
|
||||
secondsStr,
|
||||
Long.MinValue / DateTimeUtils.MICROS_PER_SECOND,
|
||||
Long.MaxValue / DateTimeUtils.MICROS_PER_SECOND) * DateTimeUtils.MICROS_PER_SECOND
|
||||
}
|
||||
def parseNanos(nanosStr: String): Long = {
|
||||
toLongWithRange("nanosecond", nanosStr, 0L, 999999999L) / DateTimeUtils.NANOS_PER_MICROS
|
||||
}
|
||||
|
||||
secondNano.split("\\.") match {
|
||||
case Array(secondsStr) => parseSeconds(secondsStr)
|
||||
case Array("", nanosStr) => parseNanos(nanosStr)
|
||||
case Array(secondsStr, nanosStr) =>
|
||||
Math.addExact(parseSeconds(secondsStr), parseNanos(nanosStr))
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
"Interval string does not match second-nano format of ss.nnnnnnnnn")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -810,7 +810,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
|
|||
checkEvaluation(new Sequence(
|
||||
Literal(Timestamp.valueOf("2018-01-01 00:00:00")),
|
||||
Literal(Timestamp.valueOf("2023-01-01 00:00:00")),
|
||||
Literal(CalendarInterval.fromYearMonthString("1-5"))),
|
||||
Literal(IntervalUtils.fromYearMonthString("1-5"))),
|
||||
Seq(
|
||||
Timestamp.valueOf("2018-01-01 00:00:00.000"),
|
||||
Timestamp.valueOf("2019-06-01 00:00:00.000"),
|
||||
|
|
@ -820,7 +820,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
|
|||
checkEvaluation(new Sequence(
|
||||
Literal(Timestamp.valueOf("2022-04-01 00:00:00")),
|
||||
Literal(Timestamp.valueOf("2017-01-01 00:00:00")),
|
||||
Literal(CalendarInterval.fromYearMonthString("1-5").negate())),
|
||||
Literal(IntervalUtils.fromYearMonthString("1-5").negate())),
|
||||
Seq(
|
||||
Timestamp.valueOf("2022-04-01 00:00:00.000"),
|
||||
Timestamp.valueOf("2020-11-01 00:00:00.000"),
|
||||
|
|
@ -894,7 +894,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
|
|||
checkEvaluation(new Sequence(
|
||||
Literal(Date.valueOf("2018-01-01")),
|
||||
Literal(Date.valueOf("2023-01-01")),
|
||||
Literal(CalendarInterval.fromYearMonthString("1-5"))),
|
||||
Literal(IntervalUtils.fromYearMonthString("1-5"))),
|
||||
Seq(
|
||||
Date.valueOf("2018-01-01"),
|
||||
Date.valueOf("2019-06-01"),
|
||||
|
|
|
|||
|
|
@ -597,7 +597,7 @@ class ExpressionParserSuite extends AnalysisTest {
|
|||
"microsecond")
|
||||
|
||||
def intervalLiteral(u: String, s: String): Literal = {
|
||||
Literal(CalendarInterval.fromUnitStrings(Array(u), Array(s)))
|
||||
Literal(IntervalUtils.fromUnitStrings(Array(u), Array(s)))
|
||||
}
|
||||
|
||||
test("intervals") {
|
||||
|
|
@ -637,7 +637,7 @@ class ExpressionParserSuite extends AnalysisTest {
|
|||
// Year-Month intervals.
|
||||
val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0")
|
||||
yearMonthValues.foreach { value =>
|
||||
val result = Literal(CalendarInterval.fromYearMonthString(value))
|
||||
val result = Literal(IntervalUtils.fromYearMonthString(value))
|
||||
checkIntervals(s"'$value' year to month", result)
|
||||
}
|
||||
|
||||
|
|
@ -650,7 +650,7 @@ class ExpressionParserSuite extends AnalysisTest {
|
|||
"-1 0:0:0",
|
||||
"1 0:0:1")
|
||||
datTimeValues.foreach { value =>
|
||||
val result = Literal(CalendarInterval.fromDayTimeString(value))
|
||||
val result = Literal(IntervalUtils.fromDayTimeString(value))
|
||||
checkIntervals(s"'$value' day to second", result)
|
||||
}
|
||||
|
||||
|
|
@ -662,7 +662,7 @@ class ExpressionParserSuite extends AnalysisTest {
|
|||
"0:0:0",
|
||||
"0:0:1")
|
||||
hourTimeValues.foreach { value =>
|
||||
val result = Literal(CalendarInterval.fromDayTimeString(value))
|
||||
val result = Literal(IntervalUtils.fromDayTimeString(value))
|
||||
checkIntervals(s"'$value' hour to second", result)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
package org.apache.spark.sql.catalyst.util
|
||||
|
||||
import org.apache.spark.SparkFunSuite
|
||||
import org.apache.spark.sql.catalyst.util.IntervalUtils.fromString
|
||||
import org.apache.spark.sql.catalyst.util.IntervalUtils.{fromDayTimeString, fromString, fromYearMonthString}
|
||||
import org.apache.spark.unsafe.types.CalendarInterval
|
||||
import org.apache.spark.unsafe.types.CalendarInterval._
|
||||
|
||||
|
|
@ -87,4 +87,65 @@ class IntervalUtilsSuite extends SparkFunSuite {
|
|||
assert(fromString(input2) == result)
|
||||
}
|
||||
}
|
||||
|
||||
test("from year-month string") {
|
||||
assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10, 0L))
|
||||
assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 + 10, 0L))
|
||||
assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10, 0L))
|
||||
|
||||
try {
|
||||
fromYearMonthString("99-15")
|
||||
fail("Expected to throw an exception for the invalid input")
|
||||
} catch {
|
||||
case e: IllegalArgumentException =>
|
||||
assert(e.getMessage.contains("month 15 outside range"))
|
||||
}
|
||||
|
||||
try {
|
||||
fromYearMonthString("9a9-15")
|
||||
fail("Expected to throw an exception for the invalid input")
|
||||
} catch {
|
||||
case e: IllegalArgumentException =>
|
||||
assert(e.getMessage.contains("Interval string does not match year-month format"))
|
||||
}
|
||||
}
|
||||
|
||||
test("from day-time string") {
|
||||
assert(fromDayTimeString("5 12:40:30.999999999") ===
|
||||
new CalendarInterval(
|
||||
0,
|
||||
5 * MICROS_PER_DAY +
|
||||
12 * MICROS_PER_HOUR +
|
||||
40 * MICROS_PER_MINUTE +
|
||||
30 * MICROS_PER_SECOND + 999999L))
|
||||
assert(fromDayTimeString("10 0:12:0.888") ===
|
||||
new CalendarInterval(
|
||||
0,
|
||||
10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE + 888 * MICROS_PER_MILLI))
|
||||
assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3 * MICROS_PER_DAY))
|
||||
|
||||
try {
|
||||
fromDayTimeString("5 30:12:20")
|
||||
fail("Expected to throw an exception for the invalid input")
|
||||
} catch {
|
||||
case e: IllegalArgumentException =>
|
||||
assert(e.getMessage.contains("hour 30 outside range"))
|
||||
}
|
||||
|
||||
try {
|
||||
fromDayTimeString("5 30-12")
|
||||
fail("Expected to throw an exception for the invalid input")
|
||||
} catch {
|
||||
case e: IllegalArgumentException =>
|
||||
assert(e.getMessage.contains("must match day-time format"))
|
||||
}
|
||||
|
||||
try {
|
||||
fromDayTimeString("5 1:12:20", "hour", "microsecond")
|
||||
fail("Expected to throw an exception for the invalid convention type")
|
||||
} catch {
|
||||
case e: IllegalArgumentException =>
|
||||
assert(e.getMessage.contains("Cannot support (interval"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue