From 77fe8a8e7cdb0eae9f66b2d10c0d35f784663547 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Fri, 18 Oct 2019 23:54:59 +0800 Subject: [PATCH] [SPARK-28420][SQL] Support the `INTERVAL` type in `date_part()` ### What changes were proposed in this pull request? The `date_part()` function can accept the `source` parameter of the `INTERVAL` type (`CalendarIntervalType`). The following values of the `field` parameter are supported: - `"MILLENNIUM"` (`"MILLENNIA"`, `"MIL"`, `"MILS"`) - number of millenniums in the given interval. It is `YEAR / 1000`. - `"CENTURY"` (`"CENTURIES"`, `"C"`, `"CENT"`) - number of centuries in the interval calculated as `YEAR / 100`. - `"DECADE"` (`"DECADES"`, `"DEC"`, `"DECS"`) - decades in the `YEAR` part of the interval calculated as `YEAR / 10`. - `"YEAR"` (`"Y"`, `"YEARS"`, `"YR"`, `"YRS"`) - years in a values of `CalendarIntervalType`. It is `MONTHS / 12`. - `"QUARTER"` (`"QTR"`) - a quarter of year calculated as `MONTHS / 3 + 1` - `"MONTH"` (`"MON"`, `"MONS"`, `"MONTHS"`) - the months part of the interval calculated as `CalendarInterval.months % 12` - `"DAY"` (`"D"`, `"DAYS"`) - total number of days in `CalendarInterval.microseconds` - `"HOUR"` (`"H"`, `"HOURS"`, `"HR"`, `"HRS"`) - the hour part of the interval. - `"MINUTE"` (`"M"`, `"MIN"`, `"MINS"`, `"MINUTES"`) - the minute part of the interval. - `"SECOND"` (`"S"`, `"SEC"`, `"SECONDS"`, `"SECS"`) - the seconds part with fractional microsecond part. - `"MILLISECONDS"` (`"MSEC"`, `"MSECS"`, `"MILLISECON"`, `"MSECONDS"`, `"MS"`) - the millisecond part of the interval with fractional microsecond part. - `"MICROSECONDS"` (`"USEC"`, `"USECS"`, `"USECONDS"`, `"MICROSECON"`, `"US"`) - the total number of microseconds in the `second`, `millisecond` and `microsecond` parts of the given interval. - `"EPOCH"` - the total number of seconds in the interval including the fractional part with microsecond precision. Here we assume 365.25 days per year (leap year every four years). For example: ```sql > SELECT date_part('days', interval 1 year 10 months 5 days); 5 > SELECT date_part('seconds', interval 30 seconds 1 milliseconds 1 microseconds); 30.001001 ``` ### Why are the changes needed? To maintain feature parity with PostgreSQL (https://www.postgresql.org/docs/11/functions-datetime.html#FUNCTIONS-DATETIME-EXTRACT) ### Does this PR introduce any user-facing change? No ### How was this patch tested? - Added new test suite `IntervalExpressionsSuite` - Add new test cases to `date_part.sql` Closes #25981 from MaxGekk/extract-from-intervals. Authored-by: Maxim Gekk Signed-off-by: Wenchen Fan --- .../expressions/datetimeExpressions.scala | 40 +- .../expressions/intervalExpressions.scala | 111 +++++ .../sql/catalyst/util/IntervalUtils.scala | 91 ++++ .../IntervalExpressionsSuite.scala | 192 +++++++ .../resources/sql-tests/inputs/date_part.sql | 75 +++ .../sql-tests/results/date_part.sql.out | 470 +++++++++++++++++- 6 files changed, 970 insertions(+), 9 deletions(-) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index cddd8c9bd6..90eb3598b2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.expressions import java.sql.Timestamp -import java.time.{DateTimeException, Instant, LocalDate, LocalDateTime, ZoneId} +import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId} import java.time.temporal.IsoFields import java.util.{Locale, TimeZone} @@ -2032,10 +2032,11 @@ object DatePart { } @ExpressionDescription( - usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp.", + usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp or interval source.", arguments = """ Arguments: - * field - selects which part of the source should be extracted. Supported string values are: + * field - selects which part of the source should be extracted. + Supported string values of `field` for dates and timestamps are: ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"), "CENTURY", ("CENTURIES", "C", "CENT"), "DECADE", ("DECADES", "DEC", "DECS"), @@ -2055,7 +2056,21 @@ object DatePart { "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"), "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"), "EPOCH"] - * source - a date (or timestamp) column from where `field` should be extracted + Supported string values of `field` for intervals are: + ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"), + "CENTURY", ("CENTURIES", "C", "CENT"), + "DECADE", ("DECADES", "DEC", "DECS"), + "YEAR", ("Y", "YEARS", "YR", "YRS"), + "QUARTER", ("QTR"), + "MONTH", ("MON", "MONS", "MONTHS"), + "DAY", ("D", "DAYS"), + "HOUR", ("H", "HOURS", "HR", "HRS"), + "MINUTE", ("M", "MIN", "MINS", "MINUTES"), + "SECOND", ("S", "SEC", "SECONDS", "SECS"), + "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"), + "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"), + "EPOCH"] + * source - a date/timestamp or interval column from where `field` should be extracted """, examples = """ Examples: @@ -2067,6 +2082,10 @@ object DatePart { 224 > SELECT _FUNC_('SECONDS', timestamp'2019-10-01 00:00:01.000001'); 1.000001 + > SELECT _FUNC_('days', interval 1 year 10 months 5 days); + 5 + > SELECT _FUNC_('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds); + 30.001001 """, since = "3.0.0") case class DatePart(field: Expression, source: Expression, child: Expression) @@ -2082,9 +2101,16 @@ case class DatePart(field: Expression, source: Expression, child: Expression) Literal(null, DoubleType) } else { val fieldStr = fieldEval.asInstanceOf[UTF8String].toString - DatePart.parseExtractField(fieldStr, source, { - throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.") - }) + val errMsg = s"Literals of type '$fieldStr' are currently not supported " + + s"for the ${source.dataType.catalogString} type." + if (source.dataType == CalendarIntervalType) { + ExtractIntervalPart.parseExtractField( + fieldStr, + source, + throw new AnalysisException(errMsg)) + } else { + DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg)) + } } }) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala new file mode 100644 index 0000000000..08360c75a4 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import java.util.Locale + +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.catalyst.util.IntervalUtils +import org.apache.spark.sql.catalyst.util.IntervalUtils._ +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.CalendarInterval + +abstract class ExtractIntervalPart( + child: Expression, + val dataType: DataType, + func: CalendarInterval => Any, + funcName: String) + extends UnaryExpression with ExpectsInputTypes with Serializable { + + override def inputTypes: Seq[AbstractDataType] = Seq(CalendarIntervalType) + + override protected def nullSafeEval(interval: Any): Any = { + func(interval.asInstanceOf[CalendarInterval]) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val iu = IntervalUtils.getClass.getName.stripSuffix("$") + defineCodeGen(ctx, ev, c => s"$iu.$funcName($c)") + } +} + +case class ExtractIntervalMillenniums(child: Expression) + extends ExtractIntervalPart(child, IntegerType, getMillenniums, "getMillenniums") + +case class ExtractIntervalCenturies(child: Expression) + extends ExtractIntervalPart(child, IntegerType, getCenturies, "getCenturies") + +case class ExtractIntervalDecades(child: Expression) + extends ExtractIntervalPart(child, IntegerType, getDecades, "getDecades") + +case class ExtractIntervalYears(child: Expression) + extends ExtractIntervalPart(child, IntegerType, getYears, "getYears") + +case class ExtractIntervalQuarters(child: Expression) + extends ExtractIntervalPart(child, ByteType, getQuarters, "getQuarters") + +case class ExtractIntervalMonths(child: Expression) + extends ExtractIntervalPart(child, ByteType, getMonths, "getMonths") + +case class ExtractIntervalDays(child: Expression) + extends ExtractIntervalPart(child, LongType, getDays, "getDays") + +case class ExtractIntervalHours(child: Expression) + extends ExtractIntervalPart(child, ByteType, getHours, "getHours") + +case class ExtractIntervalMinutes(child: Expression) + extends ExtractIntervalPart(child, ByteType, getMinutes, "getMinutes") + +case class ExtractIntervalSeconds(child: Expression) + extends ExtractIntervalPart(child, DecimalType(8, 6), getSeconds, "getSeconds") + +case class ExtractIntervalMilliseconds(child: Expression) + extends ExtractIntervalPart(child, DecimalType(8, 3), getMilliseconds, "getMilliseconds") + +case class ExtractIntervalMicroseconds(child: Expression) + extends ExtractIntervalPart(child, LongType, getMicroseconds, "getMicroseconds") + +// Number of seconds in 10000 years is 315576000001 (30 days per one month) +// which is 12 digits + 6 digits for the fractional part of seconds. +case class ExtractIntervalEpoch(child: Expression) + extends ExtractIntervalPart(child, DecimalType(18, 6), getEpoch, "getEpoch") + +object ExtractIntervalPart { + + def parseExtractField( + extractField: String, + source: Expression, + errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match { + case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => ExtractIntervalMillenniums(source) + case "CENTURY" | "CENTURIES" | "C" | "CENT" => ExtractIntervalCenturies(source) + case "DECADE" | "DECADES" | "DEC" | "DECS" => ExtractIntervalDecades(source) + case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => ExtractIntervalYears(source) + case "QUARTER" | "QTR" => ExtractIntervalQuarters(source) + case "MONTH" | "MON" | "MONS" | "MONTHS" => ExtractIntervalMonths(source) + case "DAY" | "D" | "DAYS" => ExtractIntervalDays(source) + case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => ExtractIntervalHours(source) + case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => ExtractIntervalMinutes(source) + case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => ExtractIntervalSeconds(source) + case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" => + ExtractIntervalMilliseconds(source) + case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" => + ExtractIntervalMicroseconds(source) + case "EPOCH" => ExtractIntervalEpoch(source) + case _ => errorHandleFunc + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala new file mode 100644 index 0000000000..78d188f81f --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.util + +import org.apache.spark.sql.types.Decimal +import org.apache.spark.unsafe.types.CalendarInterval + +object IntervalUtils { + val MONTHS_PER_YEAR: Int = 12 + val MONTHS_PER_QUARTER: Byte = 3 + val YEARS_PER_MILLENNIUM: Int = 1000 + val YEARS_PER_CENTURY: Int = 100 + val YEARS_PER_DECADE: Int = 10 + val MICROS_PER_HOUR: Long = DateTimeUtils.MILLIS_PER_HOUR * DateTimeUtils.MICROS_PER_MILLIS + val MICROS_PER_MINUTE: Long = DateTimeUtils.MILLIS_PER_MINUTE * DateTimeUtils.MICROS_PER_MILLIS + val DAYS_PER_MONTH: Byte = 30 + val MICROS_PER_MONTH: Long = DAYS_PER_MONTH * DateTimeUtils.SECONDS_PER_DAY + /* 365.25 days per year assumes leap year every four years */ + val MICROS_PER_YEAR: Long = (36525L * DateTimeUtils.MICROS_PER_DAY) / 100 + + def getYears(interval: CalendarInterval): Int = { + interval.months / MONTHS_PER_YEAR + } + + def getMillenniums(interval: CalendarInterval): Int = { + getYears(interval) / YEARS_PER_MILLENNIUM + } + + def getCenturies(interval: CalendarInterval): Int = { + getYears(interval) / YEARS_PER_CENTURY + } + + def getDecades(interval: CalendarInterval): Int = { + getYears(interval) / YEARS_PER_DECADE + } + + def getMonths(interval: CalendarInterval): Byte = { + (interval.months % MONTHS_PER_YEAR).toByte + } + + def getQuarters(interval: CalendarInterval): Byte = { + (getMonths(interval) / MONTHS_PER_QUARTER + 1).toByte + } + + def getDays(interval: CalendarInterval): Long = { + interval.microseconds / DateTimeUtils.MICROS_PER_DAY + } + + def getHours(interval: CalendarInterval): Byte = { + ((interval.microseconds % DateTimeUtils.MICROS_PER_DAY) / MICROS_PER_HOUR).toByte + } + + def getMinutes(interval: CalendarInterval): Byte = { + ((interval.microseconds % MICROS_PER_HOUR) / MICROS_PER_MINUTE).toByte + } + + def getMicroseconds(interval: CalendarInterval): Long = { + interval.microseconds % MICROS_PER_MINUTE + } + + def getSeconds(interval: CalendarInterval): Decimal = { + Decimal(getMicroseconds(interval), 8, 6) + } + + def getMilliseconds(interval: CalendarInterval): Decimal = { + Decimal(getMicroseconds(interval), 8, 3) + } + + // Returns total number of seconds with microseconds fractional part in the given interval. + def getEpoch(interval: CalendarInterval): Decimal = { + var result = interval.microseconds + result += MICROS_PER_YEAR * (interval.months / MONTHS_PER_YEAR) + result += MICROS_PER_MONTH * (interval.months % MONTHS_PER_YEAR) + Decimal(result, 18, 6) + } +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala new file mode 100644 index 0000000000..078ec88800 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import scala.language.implicitConversions + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.types.Decimal +import org.apache.spark.unsafe.types.CalendarInterval + +class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { + implicit def interval(s: String): Literal = { + Literal(CalendarInterval.fromString("interval " + s)) + } + + test("millenniums") { + checkEvaluation(ExtractIntervalMillenniums("0 years"), 0) + checkEvaluation(ExtractIntervalMillenniums("9999 years"), 9) + checkEvaluation(ExtractIntervalMillenniums("1000 years"), 1) + checkEvaluation(ExtractIntervalMillenniums("-2000 years"), -2) + // Microseconds part must not be taken into account + checkEvaluation(ExtractIntervalMillenniums("999 years 400 days"), 0) + // Millennium must be taken from years and months + checkEvaluation(ExtractIntervalMillenniums("999 years 12 months"), 1) + checkEvaluation(ExtractIntervalMillenniums("1000 years -1 months"), 0) + } + + test("centuries") { + checkEvaluation(ExtractIntervalCenturies("0 years"), 0) + checkEvaluation(ExtractIntervalCenturies("9999 years"), 99) + checkEvaluation(ExtractIntervalCenturies("1000 years"), 10) + checkEvaluation(ExtractIntervalCenturies("-2000 years"), -20) + // Microseconds part must not be taken into account + checkEvaluation(ExtractIntervalCenturies("99 years 400 days"), 0) + // Century must be taken from years and months + checkEvaluation(ExtractIntervalCenturies("99 years 12 months"), 1) + checkEvaluation(ExtractIntervalCenturies("100 years -1 months"), 0) + } + + test("decades") { + checkEvaluation(ExtractIntervalDecades("0 years"), 0) + checkEvaluation(ExtractIntervalDecades("9999 years"), 999) + checkEvaluation(ExtractIntervalDecades("1000 years"), 100) + checkEvaluation(ExtractIntervalDecades("-2000 years"), -200) + // Microseconds part must not be taken into account + checkEvaluation(ExtractIntervalDecades("9 years 400 days"), 0) + // Decade must be taken from years and months + checkEvaluation(ExtractIntervalDecades("9 years 12 months"), 1) + checkEvaluation(ExtractIntervalDecades("10 years -1 months"), 0) + } + + test("years") { + checkEvaluation(ExtractIntervalYears("0 years"), 0) + checkEvaluation(ExtractIntervalYears("9999 years"), 9999) + checkEvaluation(ExtractIntervalYears("1000 years"), 1000) + checkEvaluation(ExtractIntervalYears("-2000 years"), -2000) + // Microseconds part must not be taken into account + checkEvaluation(ExtractIntervalYears("9 years 400 days"), 9) + // Year must be taken from years and months + checkEvaluation(ExtractIntervalYears("9 years 12 months"), 10) + checkEvaluation(ExtractIntervalYears("10 years -1 months"), 9) + } + + test("quarters") { + checkEvaluation(ExtractIntervalQuarters("0 months"), 1.toByte) + checkEvaluation(ExtractIntervalQuarters("1 months"), 1.toByte) + checkEvaluation(ExtractIntervalQuarters("-1 months"), 1.toByte) + checkEvaluation(ExtractIntervalQuarters("2 months"), 1.toByte) + checkEvaluation(ExtractIntervalQuarters("-2 months"), 1.toByte) + checkEvaluation(ExtractIntervalQuarters("1 years -1 months"), 4.toByte) + checkEvaluation(ExtractIntervalQuarters("-1 years 1 months"), -2.toByte) + checkEvaluation(ExtractIntervalQuarters("2 years 3 months"), 2.toByte) + checkEvaluation(ExtractIntervalQuarters("-2 years -3 months"), 0.toByte) + checkEvaluation(ExtractIntervalQuarters("9999 years"), 1.toByte) + } + + test("months") { + checkEvaluation(ExtractIntervalMonths("0 year"), 0.toByte) + for (m <- -24 to 24) { + checkEvaluation(ExtractIntervalMonths(s"$m months"), (m % 12).toByte) + } + checkEvaluation(ExtractIntervalMonths("1 year 10 months"), 10.toByte) + checkEvaluation(ExtractIntervalMonths("-2 year -10 months"), -10.toByte) + checkEvaluation(ExtractIntervalMonths("9999 years"), 0.toByte) + } + + private val largeInterval: String = "9999 years 11 months " + + "31 days 11 hours 59 minutes 59 seconds 999 milliseconds 999 microseconds" + + test("days") { + checkEvaluation(ExtractIntervalDays("0 days"), 0L) + checkEvaluation(ExtractIntervalDays("1 days 100 seconds"), 1L) + checkEvaluation(ExtractIntervalDays("-1 days -100 seconds"), -1L) + checkEvaluation(ExtractIntervalDays("-365 days"), -365L) + checkEvaluation(ExtractIntervalDays("365 days"), 365L) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalDays("100 year 10 months 5 days"), 5L) + checkEvaluation(ExtractIntervalDays(largeInterval), 31L) + } + + test("hours") { + checkEvaluation(ExtractIntervalHours("0 hours"), 0.toByte) + checkEvaluation(ExtractIntervalHours("1 hour"), 1.toByte) + checkEvaluation(ExtractIntervalHours("-1 hour"), -1.toByte) + checkEvaluation(ExtractIntervalHours("23 hours"), 23.toByte) + checkEvaluation(ExtractIntervalHours("-23 hours"), -23.toByte) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalHours("100 year 10 months 10 hours"), 10.toByte) + checkEvaluation(ExtractIntervalHours(largeInterval), 11.toByte) + } + + test("minutes") { + checkEvaluation(ExtractIntervalMinutes("0 minute"), 0.toByte) + checkEvaluation(ExtractIntervalMinutes("1 minute"), 1.toByte) + checkEvaluation(ExtractIntervalMinutes("-1 minute"), -1.toByte) + checkEvaluation(ExtractIntervalMinutes("59 minute"), 59.toByte) + checkEvaluation(ExtractIntervalMinutes("-59 minute"), -59.toByte) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalMinutes("100 year 10 months 10 minutes"), 10.toByte) + checkEvaluation(ExtractIntervalMinutes(largeInterval), 59.toByte) + } + + test("seconds") { + checkEvaluation(ExtractIntervalSeconds("0 second"), Decimal(0, 8, 6)) + checkEvaluation(ExtractIntervalSeconds("1 second"), Decimal(1.0, 8, 6)) + checkEvaluation(ExtractIntervalSeconds("-1 second"), Decimal(-1.0, 8, 6)) + checkEvaluation(ExtractIntervalSeconds("1 minute 59 second"), Decimal(59.0, 8, 6)) + checkEvaluation(ExtractIntervalSeconds("-59 minutes -59 seconds"), Decimal(-59.0, 8, 6)) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalSeconds("100 year 10 months 10 seconds"), Decimal(10.0, 8, 6)) + checkEvaluation(ExtractIntervalSeconds(largeInterval), Decimal(59.999999, 8, 6)) + checkEvaluation( + ExtractIntervalSeconds("10 seconds 1 milliseconds 1 microseconds"), + Decimal(10001001, 8, 6)) + checkEvaluation(ExtractIntervalSeconds("61 seconds 1 microseconds"), Decimal(1000001, 8, 6)) + } + + test("milliseconds") { + checkEvaluation(ExtractIntervalMilliseconds("0 milliseconds"), Decimal(0, 8, 3)) + checkEvaluation(ExtractIntervalMilliseconds("1 milliseconds"), Decimal(1.0, 8, 3)) + checkEvaluation(ExtractIntervalMilliseconds("-1 milliseconds"), Decimal(-1.0, 8, 3)) + checkEvaluation( + ExtractIntervalMilliseconds("1 second 999 milliseconds"), + Decimal(1999.0, 8, 3)) + checkEvaluation( + ExtractIntervalMilliseconds("999 milliseconds 1 microsecond"), + Decimal(999.001, 8, 3)) + checkEvaluation( + ExtractIntervalMilliseconds("-1 second -999 milliseconds"), + Decimal(-1999.0, 8, 3)) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalMilliseconds("100 year 1 millisecond"), Decimal(1.0, 8, 3)) + checkEvaluation(ExtractIntervalMilliseconds(largeInterval), Decimal(59999.999, 8, 3)) + } + + test("microseconds") { + checkEvaluation(ExtractIntervalMicroseconds("0 microseconds"), 0L) + checkEvaluation(ExtractIntervalMicroseconds("1 microseconds"), 1L) + checkEvaluation(ExtractIntervalMicroseconds("-1 microseconds"), -1L) + checkEvaluation(ExtractIntervalMicroseconds("1 second 999 microseconds"), 1000999L) + checkEvaluation(ExtractIntervalMicroseconds("999 milliseconds 1 microseconds"), 999001L) + checkEvaluation(ExtractIntervalMicroseconds("-1 second -999 microseconds"), -1000999L) + // Years and months must not be taken into account + checkEvaluation(ExtractIntervalMicroseconds("11 year 1 microseconds"), 1L) + checkEvaluation(ExtractIntervalMicroseconds(largeInterval), 59999999L) + } + + test("epoch") { + checkEvaluation(ExtractIntervalEpoch("0 months"), Decimal(0.0, 18, 6)) + checkEvaluation(ExtractIntervalEpoch("10000 years"), Decimal(315576000000.0, 18, 6)) + checkEvaluation(ExtractIntervalEpoch("1 year"), Decimal(31557600.0, 18, 6)) + checkEvaluation(ExtractIntervalEpoch("-1 year"), Decimal(-31557600.0, 18, 6)) + checkEvaluation( + ExtractIntervalEpoch("1 second 1 millisecond 1 microsecond"), + Decimal(1.001001, 18, 6)) + } +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql index fd0fb50f71..a63cdafb74 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql @@ -68,3 +68,78 @@ select date_part('not_supported', c) from t; select date_part(c, c) from t; select date_part(null, c) from t; + +CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c; + +select date_part('millennium', c) from t2; +select date_part('millennia', c) from t2; +select date_part('mil', c) from t2; +select date_part('mils', c) from t2; + +select date_part('century', c) from t2; +select date_part('centuries', c) from t2; +select date_part('c', c) from t2; +select date_part('cent', c) from t2; + +select date_part('decade', c) from t2; +select date_part('decades', c) from t2; +select date_part('dec', c) from t2; +select date_part('decs', c) from t2; + +select date_part('year', c) from t2; +select date_part('y', c) from t2; +select date_part('years', c) from t2; +select date_part('yr', c) from t2; +select date_part('yrs', c) from t2; + +select date_part('quarter', c) from t2; +select date_part('qtr', c) from t2; + +select date_part('month', c) from t2; +select date_part('mon', c) from t2; +select date_part('mons', c) from t2; +select date_part('months', c) from t2; + +select date_part('day', c) from t2; +select date_part('d', c) from t2; +select date_part('days', c) from t2; + +select date_part('hour', c) from t2; +select date_part('h', c) from t2; +select date_part('hours', c) from t2; +select date_part('hr', c) from t2; +select date_part('hrs', c) from t2; + +select date_part('minute', c) from t2; +select date_part('m', c) from t2; +select date_part('min', c) from t2; +select date_part('mins', c) from t2; +select date_part('minutes', c) from t2; + +select date_part('second', c) from t2; +select date_part('s', c) from t2; +select date_part('sec', c) from t2; +select date_part('seconds', c) from t2; +select date_part('secs', c) from t2; + +select date_part('milliseconds', c) from t2; +select date_part('msec', c) from t2; +select date_part('msecs', c) from t2; +select date_part('millisecon', c) from t2; +select date_part('mseconds', c) from t2; +select date_part('ms', c) from t2; + +select date_part('microseconds', c) from t2; +select date_part('usec', c) from t2; +select date_part('usecs', c) from t2; +select date_part('useconds', c) from t2; +select date_part('microsecon', c) from t2; +select date_part('us', c) from t2; + +select date_part('epoch', c) from t2; + +select date_part('not_supported', c) from t2; + +select date_part(c, c) from t2; + +select date_part(null, c) from t2; diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out index bbdfe6cc22..8f4edf1960 100644 --- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 52 +-- Number of queries: 110 -- !query 0 @@ -400,7 +400,7 @@ select date_part('not_supported', c) from t struct<> -- !query 49 output org.apache.spark.sql.AnalysisException -Literals of type 'not_supported' are currently not supported.;; line 1 pos 7 +Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7 -- !query 50 @@ -418,3 +418,469 @@ select date_part(null, c) from t struct -- !query 51 output NULL + + +-- !query 52 +CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c +-- !query 52 schema +struct<> +-- !query 52 output + + + +-- !query 53 +select date_part('millennium', c) from t2 +-- !query 53 schema +struct +-- !query 53 output +1 + + +-- !query 54 +select date_part('millennia', c) from t2 +-- !query 54 schema +struct +-- !query 54 output +1 + + +-- !query 55 +select date_part('mil', c) from t2 +-- !query 55 schema +struct +-- !query 55 output +1 + + +-- !query 56 +select date_part('mils', c) from t2 +-- !query 56 schema +struct +-- !query 56 output +1 + + +-- !query 57 +select date_part('century', c) from t2 +-- !query 57 schema +struct +-- !query 57 output +10 + + +-- !query 58 +select date_part('centuries', c) from t2 +-- !query 58 schema +struct +-- !query 58 output +10 + + +-- !query 59 +select date_part('c', c) from t2 +-- !query 59 schema +struct +-- !query 59 output +10 + + +-- !query 60 +select date_part('cent', c) from t2 +-- !query 60 schema +struct +-- !query 60 output +10 + + +-- !query 61 +select date_part('decade', c) from t2 +-- !query 61 schema +struct +-- !query 61 output +101 + + +-- !query 62 +select date_part('decades', c) from t2 +-- !query 62 schema +struct +-- !query 62 output +101 + + +-- !query 63 +select date_part('dec', c) from t2 +-- !query 63 schema +struct +-- !query 63 output +101 + + +-- !query 64 +select date_part('decs', c) from t2 +-- !query 64 schema +struct +-- !query 64 output +101 + + +-- !query 65 +select date_part('year', c) from t2 +-- !query 65 schema +struct +-- !query 65 output +1010 + + +-- !query 66 +select date_part('y', c) from t2 +-- !query 66 schema +struct +-- !query 66 output +1010 + + +-- !query 67 +select date_part('years', c) from t2 +-- !query 67 schema +struct +-- !query 67 output +1010 + + +-- !query 68 +select date_part('yr', c) from t2 +-- !query 68 schema +struct +-- !query 68 output +1010 + + +-- !query 69 +select date_part('yrs', c) from t2 +-- !query 69 schema +struct +-- !query 69 output +1010 + + +-- !query 70 +select date_part('quarter', c) from t2 +-- !query 70 schema +struct +-- !query 70 output +4 + + +-- !query 71 +select date_part('qtr', c) from t2 +-- !query 71 schema +struct +-- !query 71 output +4 + + +-- !query 72 +select date_part('month', c) from t2 +-- !query 72 schema +struct +-- !query 72 output +9 + + +-- !query 73 +select date_part('mon', c) from t2 +-- !query 73 schema +struct +-- !query 73 output +9 + + +-- !query 74 +select date_part('mons', c) from t2 +-- !query 74 schema +struct +-- !query 74 output +9 + + +-- !query 75 +select date_part('months', c) from t2 +-- !query 75 schema +struct +-- !query 75 output +9 + + +-- !query 76 +select date_part('day', c) from t2 +-- !query 76 schema +struct +-- !query 76 output +8 + + +-- !query 77 +select date_part('d', c) from t2 +-- !query 77 schema +struct +-- !query 77 output +8 + + +-- !query 78 +select date_part('days', c) from t2 +-- !query 78 schema +struct +-- !query 78 output +8 + + +-- !query 79 +select date_part('hour', c) from t2 +-- !query 79 schema +struct +-- !query 79 output +7 + + +-- !query 80 +select date_part('h', c) from t2 +-- !query 80 schema +struct +-- !query 80 output +7 + + +-- !query 81 +select date_part('hours', c) from t2 +-- !query 81 schema +struct +-- !query 81 output +7 + + +-- !query 82 +select date_part('hr', c) from t2 +-- !query 82 schema +struct +-- !query 82 output +7 + + +-- !query 83 +select date_part('hrs', c) from t2 +-- !query 83 schema +struct +-- !query 83 output +7 + + +-- !query 84 +select date_part('minute', c) from t2 +-- !query 84 schema +struct +-- !query 84 output +6 + + +-- !query 85 +select date_part('m', c) from t2 +-- !query 85 schema +struct +-- !query 85 output +6 + + +-- !query 86 +select date_part('min', c) from t2 +-- !query 86 schema +struct +-- !query 86 output +6 + + +-- !query 87 +select date_part('mins', c) from t2 +-- !query 87 schema +struct +-- !query 87 output +6 + + +-- !query 88 +select date_part('minutes', c) from t2 +-- !query 88 schema +struct +-- !query 88 output +6 + + +-- !query 89 +select date_part('second', c) from t2 +-- !query 89 schema +struct +-- !query 89 output +5.004003 + + +-- !query 90 +select date_part('s', c) from t2 +-- !query 90 schema +struct +-- !query 90 output +5.004003 + + +-- !query 91 +select date_part('sec', c) from t2 +-- !query 91 schema +struct +-- !query 91 output +5.004003 + + +-- !query 92 +select date_part('seconds', c) from t2 +-- !query 92 schema +struct +-- !query 92 output +5.004003 + + +-- !query 93 +select date_part('secs', c) from t2 +-- !query 93 schema +struct +-- !query 93 output +5.004003 + + +-- !query 94 +select date_part('milliseconds', c) from t2 +-- !query 94 schema +struct +-- !query 94 output +5004.003 + + +-- !query 95 +select date_part('msec', c) from t2 +-- !query 95 schema +struct +-- !query 95 output +5004.003 + + +-- !query 96 +select date_part('msecs', c) from t2 +-- !query 96 schema +struct +-- !query 96 output +5004.003 + + +-- !query 97 +select date_part('millisecon', c) from t2 +-- !query 97 schema +struct +-- !query 97 output +5004.003 + + +-- !query 98 +select date_part('mseconds', c) from t2 +-- !query 98 schema +struct +-- !query 98 output +5004.003 + + +-- !query 99 +select date_part('ms', c) from t2 +-- !query 99 schema +struct +-- !query 99 output +5004.003 + + +-- !query 100 +select date_part('microseconds', c) from t2 +-- !query 100 schema +struct +-- !query 100 output +5004003 + + +-- !query 101 +select date_part('usec', c) from t2 +-- !query 101 schema +struct +-- !query 101 output +5004003 + + +-- !query 102 +select date_part('usecs', c) from t2 +-- !query 102 schema +struct +-- !query 102 output +5004003 + + +-- !query 103 +select date_part('useconds', c) from t2 +-- !query 103 schema +struct +-- !query 103 output +5004003 + + +-- !query 104 +select date_part('microsecon', c) from t2 +-- !query 104 schema +struct +-- !query 104 output +5004003 + + +-- !query 105 +select date_part('us', c) from t2 +-- !query 105 schema +struct +-- !query 105 output +5004003 + + +-- !query 106 +select date_part('epoch', c) from t2 +-- !query 106 schema +struct +-- !query 106 output +31873892788.332003 + + +-- !query 107 +select date_part('not_supported', c) from t2 +-- !query 107 schema +struct<> +-- !query 107 output +org.apache.spark.sql.AnalysisException +Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7 + + +-- !query 108 +select date_part(c, c) from t2 +-- !query 108 schema +struct<> +-- !query 108 output +org.apache.spark.sql.AnalysisException +The field parameter needs to be a foldable string value.;; line 1 pos 7 + + +-- !query 109 +select date_part(null, c) from t2 +-- !query 109 schema +struct +-- !query 109 output +NULL