[SPARK-29864][SQL][FOLLOWUP] Reference the config for the old behavior in error message

### What changes were proposed in this pull request?
Follow up work for SPARK-29864, reference the config  `spark.sql.legacy.fromDayTimeString.enabled` in error message.

### Why are the changes needed?
For better usability.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests.

Closes #27464 from xuanyuanking/SPARK-29864-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
Yuanjian Li 2020-02-05 11:19:42 -08:00 committed by Dongjoon Hyun
parent 9d90c8b898
commit 4938905a1c
4 changed files with 29 additions and 24 deletions

View file

@ -176,6 +176,9 @@ object IntervalUtils {
private val dayTimePatternLegacy =
"^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r
private val fallbackNotice = s"set ${SQLConf.LEGACY_FROM_DAYTIME_STRING.key} to true " +
"to restore the behavior before Spark 3.0."
/**
* Legacy method of parsing a string in a day-time format. It ignores the `from` bound,
* and takes into account only the `to` bound by truncating the result. For example,
@ -195,7 +198,8 @@ object IntervalUtils {
require(input != null, "Interval day-time string must be not null")
assert(input.length == input.trim.length)
val m = dayTimePatternLegacy.pattern.matcher(input)
require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input")
require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " +
s"$fallbackNotice")
try {
val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1
@ -296,7 +300,8 @@ object IntervalUtils {
require(regexp.isDefined, s"Cannot support (interval '$input' $from to $to) expression")
val pattern = regexp.get.pattern
val m = pattern.matcher(input)
require(m.matches, s"Interval string must match day-time format of '$pattern': $input")
require(m.matches, s"Interval string must match day-time format of '$pattern': $input, " +
s"$fallbackNotice")
var micros: Long = 0L
var days: Int = 0
unitsRange(to, from).foreach {

View file

@ -320,7 +320,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 15:40:32.99899999' day to hour
@ -334,7 +334,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 15:40:32.99899999' day to minute
@ -348,7 +348,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40:32.99899999' hour to minute
@ -362,7 +362,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40.99899999' hour to second
@ -376,7 +376,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40' hour to second
@ -390,7 +390,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 40:32.99899999' minute to second

View file

@ -314,7 +314,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 15:40:32.99899999' day to hour
@ -328,7 +328,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 15:40:32.99899999' day to minute
@ -342,7 +342,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40:32.99899999' hour to minute
@ -356,7 +356,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40.99899999' hour to second
@ -370,7 +370,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '15:40' hour to second
@ -384,7 +384,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
select interval '20 40:32.99899999' minute to second

View file

@ -105,7 +105,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03' day to hour
@ -119,7 +119,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03:04' day to hour
@ -141,7 +141,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03:04' day to minute
@ -155,7 +155,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03' day to second
@ -177,7 +177,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03' hour to minute
@ -191,7 +191,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03:04' hour to minute
@ -205,7 +205,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03' hour to second
@ -219,7 +219,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03:04' hour to second
@ -233,7 +233,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03' minute to second
@ -247,7 +247,7 @@ struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
== SQL ==
SELECT interval '1 2:03:04' minute to second