From 4938905a1c047e367c066e39dce8232bfcff14f1 Mon Sep 17 00:00:00 2001 From: Yuanjian Li Date: Wed, 5 Feb 2020 11:19:42 -0800 Subject: [PATCH] [SPARK-29864][SQL][FOLLOWUP] Reference the config for the old behavior in error message ### What changes were proposed in this pull request? Follow up work for SPARK-29864, reference the config `spark.sql.legacy.fromDayTimeString.enabled` in error message. ### Why are the changes needed? For better usability. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Existing tests. Closes #27464 from xuanyuanking/SPARK-29864-follow. Authored-by: Yuanjian Li Signed-off-by: Dongjoon Hyun --- .../sql/catalyst/util/IntervalUtils.scala | 9 +++++++-- .../sql-tests/results/ansi/interval.sql.out | 12 +++++------ .../sql-tests/results/interval.sql.out | 12 +++++------ .../results/postgreSQL/interval.sql.out | 20 +++++++++---------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala index 7692299a46..2d98384363 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala @@ -176,6 +176,9 @@ object IntervalUtils { private val dayTimePatternLegacy = "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r + private val fallbackNotice = s"set ${SQLConf.LEGACY_FROM_DAYTIME_STRING.key} to true " + + "to restore the behavior before Spark 3.0." + /** * Legacy method of parsing a string in a day-time format. It ignores the `from` bound, * and takes into account only the `to` bound by truncating the result. For example, @@ -195,7 +198,8 @@ object IntervalUtils { require(input != null, "Interval day-time string must be not null") assert(input.length == input.trim.length) val m = dayTimePatternLegacy.pattern.matcher(input) - require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input") + require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " + + s"$fallbackNotice") try { val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1 @@ -296,7 +300,8 @@ object IntervalUtils { require(regexp.isDefined, s"Cannot support (interval '$input' $from to $to) expression") val pattern = regexp.get.pattern val m = pattern.matcher(input) - require(m.matches, s"Interval string must match day-time format of '$pattern': $input") + require(m.matches, s"Interval string must match day-time format of '$pattern': $input, " + + s"$fallbackNotice") var micros: Long = 0L var days: Int = 0 unitsRange(to, from).foreach { diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index ab6130da86..f37049064d 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -320,7 +320,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 15:40:32.99899999' day to hour @@ -334,7 +334,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 15:40:32.99899999' day to minute @@ -348,7 +348,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40:32.99899999' hour to minute @@ -362,7 +362,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40.99899999' hour to second @@ -376,7 +376,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40' hour to second @@ -390,7 +390,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 40:32.99899999' minute to second diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 8f523a35f3..94b4f15815 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -314,7 +314,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 15:40:32.99899999' day to hour @@ -328,7 +328,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 15:40:32.99899999' day to minute @@ -342,7 +342,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 15:40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40:32.99899999' hour to minute @@ -356,7 +356,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40.99899999' hour to second @@ -370,7 +370,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '15:40' hour to second @@ -384,7 +384,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == select interval '20 40:32.99899999' minute to second diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out index 4bd846d3ff..62d47410aa 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out @@ -105,7 +105,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 1 2:03(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03' day to hour @@ -119,7 +119,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 1 2:03:04(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03:04' day to hour @@ -141,7 +141,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 1 2:03:04(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03:04' day to minute @@ -155,7 +155,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d+) (?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03' day to second @@ -177,7 +177,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 1 2:03(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03' hour to minute @@ -191,7 +191,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 1 2:03:04(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03:04' hour to minute @@ -205,7 +205,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03' hour to second @@ -219,7 +219,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03:04' hour to second @@ -233,7 +233,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03' minute to second @@ -247,7 +247,7 @@ struct<> -- !query output org.apache.spark.sql.catalyst.parser.ParseException -requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16) +requirement failed: Interval string must match day-time format of '^(?[+|-])?(?\d{1,2}):(?(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16) == SQL == SELECT interval '1 2:03:04' minute to second