diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json index 8d36fbd377..e5f9b0cb7a 100644 --- a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json +++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json @@ -54,6 +54,10 @@ "tz" : "Africa/Cairo", "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ], "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ] +}, { + "tz" : "Africa/Casablanca", + "switches" : [ -62135594980, -59006359780, -55850686180, -52695012580, -46383578980, -43227905380, -40072231780, -33760798180, -30605124580, -27449450980, -21138017380, -17982343780, -14826670180, -12220068580, -12219982180, -12219895780, -12219809380, -12219722980, -12219636580, -12219550180, -12219463780, -12219377380, -12219290980, -2208988800 ], + "diffs" : [ -174620, -88220, -1820, 84580, 170980, 257380, 343780, 430180, 516580, 602980, 689380, 775780, 862180, 775780, 689380, 602980, 516580, 430180, 343780, 257380, 170980, 84580, -1820, 0 ] }, { "tz" : "Africa/Ceuta", "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12220069124, -12219982724, -12219896324, -12219809924, -12219723524, -12219637124, -12219550724, -12219464324, -12219377924, -12219291524, -2208988800 ], @@ -78,6 +82,10 @@ "tz" : "Africa/Douala", "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] +}, { + "tz" : "Africa/El_Aaiun", + "switches" : [ -62135593632, -59006358432, -55850684832, -52695011232, -46383577632, -43227904032, -40072230432, -33760796832, -30605123232, -27449449632, -21138016032, -17982342432, -14826668832, -12220067232, -12219980832, -12219894432, -12219808032, -12219721632, -12219635232, -12219548832, -12219462432, -12219376032, -12219289632, -2208988800 ], + "diffs" : [ -175968, -89568, -3168, 83232, 169632, 256032, 342432, 428832, 515232, 601632, 688032, 774432, 860832, 774432, 688032, 601632, 515232, 428832, 342432, 256032, 169632, 83232, -3168, 0 ] }, { "tz" : "Africa/Freetown", "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], @@ -1250,6 +1258,10 @@ "tz" : "Asia/Tbilisi", "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12220081151, -12219994751, -12219908351, -12219821951, -12219735551, -12219649151, -12219562751, -12219476351, -12219389951, -12219303551, -2208988800 ], "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, 773951, 687551, 601151, 514751, 428351, 341951, 255551, 169151, 82751, -3649, 0 ] +}, { + "tz" : "Asia/Tehran", + "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ], + "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ] }, { "tz" : "Asia/Tel_Aviv", "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ], @@ -2002,6 +2014,10 @@ "tz" : "Indian/Reunion", "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12220083712, -12219997312, -12219910912, -12219824512, -12219738112, -12219651712, -12219565312, -12219478912, -12219392512, -12219306112, -2208988800 ], "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, 0 ] +}, { + "tz" : "Iran", + "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ], + "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ] }, { "tz" : "Israel", "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ], diff --git a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json index b733ac43cb..eddc16f9ae 100644 --- a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json +++ b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json @@ -54,6 +54,10 @@ "tz" : "Africa/Cairo", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ] +}, { + "tz" : "Africa/Casablanca", + "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], + "diffs" : [ 174620, 88220, 1820, -84580, -170980, -257380, -343780, -430180, -516580, -602980, -689380, -775780, -862180, 1820, 0 ] }, { "tz" : "Africa/Ceuta", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -78,6 +82,10 @@ "tz" : "Africa/Douala", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] +}, { + "tz" : "Africa/El_Aaiun", + "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], + "diffs" : [ 175968, 89568, 3168, -83232, -169632, -256032, -342432, -428832, -515232, -601632, -688032, -774432, -860832, 3168, 0 ] }, { "tz" : "Africa/Freetown", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -1250,6 +1258,10 @@ "tz" : "Asia/Tbilisi", "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ], "diffs" : [ 176449, 90049, 3649, -82751, -169151, -255551, -341951, -428351, -514751, -601151, -687551, -773951, -860351, 3649, 0 ] +}, { + "tz" : "Asia/Tehran", + "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ], + "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ] }, { "tz" : "Asia/Tel_Aviv", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ], @@ -2002,6 +2014,10 @@ "tz" : "Indian/Reunion", "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ], "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 0 ] +}, { + "tz" : "Iran", + "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ], + "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ] }, { "tz" : "Israel", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ], diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala index f354bf9673..cb5f8e43d7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala @@ -256,22 +256,16 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper { case class RebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long]) val result = new ArrayBuffer[RebaseRecord]() - // The time zones are excluded because: - // 1. Julian to Gregorian rebasing doesn't match to the opposite rebasing from - // Gregorian to Julian rebasing. - // 2. Linear searching over switch points might be slow. - // 3. Results after the end time point 2100-01-01 are wrong. - // See SPARK-31385 - val blacklist = Set("Asia/Tehran", "Iran", "Africa/Casablanca", "Africa/El_Aaiun") ALL_TIMEZONES - .filterNot(zid => blacklist.contains(zid.getId)) .sortBy(_.getId) .foreach { zid => withDefaultTimeZone(zid) { val start = adjustFunc(instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0) .atZone(zid) .toInstant)) - val end = adjustFunc(instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0) + // sun.util.calendar.ZoneInfo resolves DST after 2037 year incorrectly. + // See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8073446 + val end = adjustFunc(instantToMicros(LocalDateTime.of(2037, 1, 1, 0, 0, 0) .atZone(zid) .toInstant))