From c26976fe148a2a59cec2f399484be73d08fb6b7f Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 8 Sep 2017 09:31:13 +0800 Subject: [PATCH] [SPARK-21939][TEST] Use TimeLimits instead of Timeouts Since ScalaTest 3.0.0, `org.scalatest.concurrent.Timeouts` is deprecated. This PR replaces the deprecated one with `org.scalatest.concurrent.TimeLimits`. ```scala -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ ``` Pass the existing test suites. Author: Dongjoon Hyun Closes #19150 from dongjoon-hyun/SPARK-21939. Change-Id: I1a1b07f1b97e51e2263dfb34b7eaaa099b2ded5e --- core/src/test/scala/org/apache/spark/DistributedSuite.scala | 2 +- core/src/test/scala/org/apache/spark/DriverSuite.scala | 4 ++-- core/src/test/scala/org/apache/spark/UnpersistSuite.scala | 2 +- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 4 ++-- .../scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala | 4 ++-- .../scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala | 4 ++-- .../scheduler/OutputCommitCoordinatorIntegrationSuite.scala | 4 ++-- .../scala/org/apache/spark/storage/BlockManagerSuite.scala | 2 +- .../test/scala/org/apache/spark/util/EventLoopSuite.scala | 4 ++-- .../execution/streaming/ProcessingTimeExecutorSuite.scala | 2 +- .../scala/org/apache/spark/sql/streaming/StreamTest.scala | 5 +++-- .../org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala | 4 ++-- .../spark/streaming/receiver/BlockGeneratorSuite.scala | 4 +++- 13 files changed, 24 insertions(+), 21 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/DistributedSuite.scala b/core/src/test/scala/org/apache/spark/DistributedSuite.scala index 2337238494..bea67b71a5 100644 --- a/core/src/test/scala/org/apache/spark/DistributedSuite.scala +++ b/core/src/test/scala/org/apache/spark/DistributedSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark import org.scalatest.Matchers -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ import org.scalatest.time.{Millis, Span} import org.apache.spark.security.EncryptionFunSuite diff --git a/core/src/test/scala/org/apache/spark/DriverSuite.scala b/core/src/test/scala/org/apache/spark/DriverSuite.scala index 454b7e607a..be80d278fc 100644 --- a/core/src/test/scala/org/apache/spark/DriverSuite.scala +++ b/core/src/test/scala/org/apache/spark/DriverSuite.scala @@ -19,13 +19,13 @@ package org.apache.spark import java.io.File -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.prop.TableDrivenPropertyChecks._ import org.scalatest.time.SpanSugar._ import org.apache.spark.util.Utils -class DriverSuite extends SparkFunSuite with Timeouts { +class DriverSuite extends SparkFunSuite with TimeLimits { ignore("driver should exit after finishing without cleanup (SPARK-530)") { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) diff --git a/core/src/test/scala/org/apache/spark/UnpersistSuite.scala b/core/src/test/scala/org/apache/spark/UnpersistSuite.scala index 09e21646ee..bc3f58cf2a 100644 --- a/core/src/test/scala/org/apache/spark/UnpersistSuite.scala +++ b/core/src/test/scala/org/apache/spark/UnpersistSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ import org.scalatest.time.{Millis, Span} class UnpersistSuite extends SparkFunSuite with LocalSparkContext { diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 7400ceb802..4d69ce844d 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -31,7 +31,7 @@ import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path} import org.scalatest.{BeforeAndAfterEach, Matchers} -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.time.SpanSugar._ import org.apache.spark._ @@ -97,7 +97,7 @@ class SparkSubmitSuite with Matchers with BeforeAndAfterEach with ResetSystemProperties - with Timeouts + with TimeLimits with TestPrematureExit { override def beforeEach() { diff --git a/core/src/test/scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala index 8f639eef46..f4be8eaef7 100644 --- a/core/src/test/scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala +++ b/core/src/test/scala/org/apache/spark/rdd/AsyncRDDActionsSuite.scala @@ -24,13 +24,13 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.Duration import org.scalatest.BeforeAndAfterAll -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.time.SpanSugar._ import org.apache.spark._ import org.apache.spark.util.ThreadUtils -class AsyncRDDActionsSuite extends SparkFunSuite with BeforeAndAfterAll with Timeouts { +class AsyncRDDActionsSuite extends SparkFunSuite with BeforeAndAfterAll with TimeLimits { @transient private var sc: SparkContext = _ diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index 3b5df657d4..703fc1b34c 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -25,7 +25,7 @@ import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map} import scala.language.reflectiveCalls import scala.util.control.NonFatal -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.time.SpanSugar._ import org.apache.spark._ @@ -98,7 +98,7 @@ class MyRDD( class DAGSchedulerSuiteDummyException extends Exception -class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with Timeouts { +class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLimits { import DAGSchedulerSuite._ diff --git a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorIntegrationSuite.scala index 32cdf16dd3..a27dadcf49 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorIntegrationSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorIntegrationSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.scheduler import org.apache.hadoop.mapred.{FileOutputCommitter, TaskAttemptContext} -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.time.{Seconds, Span} import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite, TaskContext} @@ -32,7 +32,7 @@ import org.apache.spark.util.Utils class OutputCommitCoordinatorIntegrationSuite extends SparkFunSuite with LocalSparkContext - with Timeouts { + with TimeLimits { override def beforeAll(): Unit = { super.beforeAll() diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala index 8dd70fcb2f..cfe89fde63 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala @@ -33,7 +33,7 @@ import org.mockito.{Matchers => mc} import org.mockito.Mockito.{mock, times, verify, when} import org.scalatest._ import org.scalatest.concurrent.Eventually._ -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ import org.apache.spark._ import org.apache.spark.broadcast.BroadcastManager diff --git a/core/src/test/scala/org/apache/spark/util/EventLoopSuite.scala b/core/src/test/scala/org/apache/spark/util/EventLoopSuite.scala index 6f7dddd4f7..f4f8388f5f 100644 --- a/core/src/test/scala/org/apache/spark/util/EventLoopSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/EventLoopSuite.scala @@ -24,11 +24,11 @@ import scala.concurrent.duration._ import scala.language.postfixOps import org.scalatest.concurrent.Eventually._ -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.apache.spark.SparkFunSuite -class EventLoopSuite extends SparkFunSuite with Timeouts { +class EventLoopSuite extends SparkFunSuite with TimeLimits { test("EventLoop") { val buffer = new ConcurrentLinkedQueue[Int] diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala index 007554a83f..519e3c01af 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/ProcessingTimeExecutorSuite.scala @@ -24,7 +24,7 @@ import scala.collection.mutable import org.eclipse.jetty.util.ConcurrentHashSet import org.scalatest.concurrent.Eventually import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkFunSuite diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala index b2c42eef88..4f8764060d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala @@ -27,7 +27,7 @@ import scala.util.Random import scala.util.control.NonFatal import org.scalatest.{Assertions, BeforeAndAfterAll} -import org.scalatest.concurrent.{Eventually, Timeouts} +import org.scalatest.concurrent.{Eventually, Signaler, ThreadSignaler, TimeLimits} import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.exceptions.TestFailedDueToTimeoutException import org.scalatest.time.Span @@ -67,8 +67,9 @@ import org.apache.spark.util.{Clock, SystemClock, Utils} * avoid hanging forever in the case of failures. However, individual suites can change this * by overriding `streamingTimeout`. */ -trait StreamTest extends QueryTest with SharedSQLContext with Timeouts with BeforeAndAfterAll { +trait StreamTest extends QueryTest with SharedSQLContext with TimeLimits with BeforeAndAfterAll { + implicit val defaultSignaler: Signaler = ThreadSignaler override def afterAll(): Unit = { super.afterAll() StateStore.stop() // stop the state store maintenance thread and unload store providers diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index 17715465d3..be6aa6d8dc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -26,7 +26,7 @@ import scala.tools.nsc.Properties import org.apache.hadoop.fs.Path import org.scalatest.{BeforeAndAfterEach, Matchers} -import org.scalatest.concurrent.Timeouts +import org.scalatest.concurrent.TimeLimits import org.scalatest.exceptions.TestFailedDueToTimeoutException import org.scalatest.time.SpanSugar._ @@ -50,7 +50,7 @@ class HiveSparkSubmitSuite with Matchers with BeforeAndAfterEach with ResetSystemProperties - with Timeouts { + with TimeLimits { // TODO: rewrite these or mark them as slow tests to be run sparingly diff --git a/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala index 4f41b9d0a0..898da4445e 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/receiver/BlockGeneratorSuite.scala @@ -24,8 +24,9 @@ import scala.collection.mutable import org.scalatest.BeforeAndAfter import org.scalatest.Matchers._ +import org.scalatest.concurrent.{Signaler, ThreadSignaler} import org.scalatest.concurrent.Eventually._ -import org.scalatest.concurrent.Timeouts._ +import org.scalatest.concurrent.TimeLimits._ import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkConf, SparkException, SparkFunSuite} @@ -34,6 +35,7 @@ import org.apache.spark.util.ManualClock class BlockGeneratorSuite extends SparkFunSuite with BeforeAndAfter { + implicit val defaultSignaler: Signaler = ThreadSignaler private val blockIntervalMs = 10 private val conf = new SparkConf().set("spark.streaming.blockInterval", s"${blockIntervalMs}ms") @volatile private var blockGenerator: BlockGenerator = null