[SPARK-36025][SQL][TESTS] Reduce the run time of DateExpressionsSuite

### What changes were proposed in this pull request?

Some of the test cases in `DateExpressionsSuite` are quite slow:

- `Hour`: 24s
- `Minute`: 26s
- `Day / DayOfMonth`: 8s
- `Year`: 4s

Each test case has a large loop. We should improve them.

### Why are the changes needed?

Save test running time

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Verified the run times on local:
- `Hour`: 2s
- `Minute`: 3.2
- `Day / DayOfMonth`:0.5s
- `Year`: 2s

Total reduced time: 54.3s

Closes #33229 from gengliangwang/improveTest.

Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
This commit is contained in:
Gengliang Wang 2021-07-06 20:17:02 +08:00
parent d8e91eb2f6
commit d5d1222686

View file

@ -25,7 +25,9 @@ import java.time.temporal.ChronoUnit
import java.util.{Calendar, Locale, TimeZone} import java.util.{Calendar, Locale, TimeZone}
import java.util.concurrent.TimeUnit._ import java.util.concurrent.TimeUnit._
import scala.language.postfixOps
import scala.reflect.ClassTag import scala.reflect.ClassTag
import scala.util.Random
import org.apache.spark.{SparkFunSuite, SparkUpgradeException} import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.InternalRow
@ -122,8 +124,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
(2000 to 2002).foreach { y => (2000 to 2002).foreach { y =>
(0 to 11 by 11).foreach { m => (0 to 11 by 11).foreach { m =>
c.set(y, m, 28) c.set(y, m, 28)
(0 to 5 * 24).foreach { i => (0 to 12).foreach { i =>
c.add(Calendar.HOUR_OF_DAY, 1) c.add(Calendar.HOUR_OF_DAY, 10)
checkEvaluation(Year(Literal(new Date(c.getTimeInMillis))), checkEvaluation(Year(Literal(new Date(c.getTimeInMillis))),
c.get(Calendar.YEAR)) c.get(Calendar.YEAR))
} }
@ -195,8 +197,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val c = Calendar.getInstance() val c = Calendar.getInstance()
(1999 to 2000).foreach { y => (1999 to 2000).foreach { y =>
c.set(y, 0, 1, 0, 0, 0) c.set(y, 0, 1, 0, 0, 0)
(0 to 365).foreach { d => val random = new Random(System.nanoTime)
c.add(Calendar.DATE, 1) random.shuffle(0 to 365 toList).take(10).foreach { d =>
c.set(Calendar.DAY_OF_YEAR, d)
checkEvaluation(DayOfMonth(Literal(new Date(c.getTimeInMillis))), checkEvaluation(DayOfMonth(Literal(new Date(c.getTimeInMillis))),
c.get(Calendar.DAY_OF_MONTH)) c.get(Calendar.DAY_OF_MONTH))
} }
@ -332,19 +335,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val timeZoneId = Option(zid.getId) val timeZoneId = Option(zid.getId)
c.setTimeZone(TimeZone.getTimeZone(zid)) c.setTimeZone(TimeZone.getTimeZone(zid))
(0 to 24 by 5).foreach { h => (0 to 24 by 5).foreach { h =>
(0 to 60 by 29).foreach { m => // validate timestamp with local time zone
(0 to 60 by 29).foreach { s => c.set(2015, 18, 3, h, 29, 59)
// validate timestamp with local time zone checkEvaluation(
c.set(2015, 18, 3, h, m, s) Hour(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
checkEvaluation( c.get(Calendar.HOUR_OF_DAY))
Hour(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
c.get(Calendar.HOUR_OF_DAY))
// validate timestamp without time zone // validate timestamp without time zone
val localDateTime = LocalDateTime.of(2015, 1, 3, h, m, s) val localDateTime = LocalDateTime.of(2015, 1, 3, h, 29, 59)
checkEvaluation(Hour(Literal(localDateTime), timeZoneId), h) checkEvaluation(Hour(Literal(localDateTime), timeZoneId), h)
}
}
} }
Seq(TimestampType, TimestampNTZType).foreach { dt => Seq(TimestampType, TimestampNTZType).foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen( checkConsistencyBetweenInterpretedAndCodegen(
@ -367,17 +366,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val timeZoneId = Option(zid.getId) val timeZoneId = Option(zid.getId)
c.setTimeZone(TimeZone.getTimeZone(zid)) c.setTimeZone(TimeZone.getTimeZone(zid))
(0 to 59 by 5).foreach { m => (0 to 59 by 5).foreach { m =>
(0 to 59 by 15).foreach { s => // validate timestamp with local time zone
// validate timestamp with local time zone c.set(2015, 18, 3, 3, m, 3)
c.set(2015, 18, 3, 3, m, s) checkEvaluation(
checkEvaluation( Minute(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
Minute(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId), c.get(Calendar.MINUTE))
c.get(Calendar.MINUTE))
// validate timestamp without time zone // validate timestamp without time zone
val localDateTime = LocalDateTime.of(2015, 1, 3, 3, m, s) val localDateTime = LocalDateTime.of(2015, 1, 3, 3, m, 3)
checkEvaluation(Minute(Literal(localDateTime), timeZoneId), m) checkEvaluation(Minute(Literal(localDateTime), timeZoneId), m)
}
} }
Seq(TimestampType, TimestampNTZType).foreach { dt => Seq(TimestampType, TimestampNTZType).foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen( checkConsistencyBetweenInterpretedAndCodegen(