ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[MINOR][SQL][TESTS] Create tables once in JDBC tests

Browse source 
### What changes were proposed in this pull request?
In PR, I propose to create input tables once before executing tests in `JDBCSuite` and `JdbcRDDSuite`. Currently, the table are created before every test in the test suites.

### Why are the changes needed?
This speed up the test suites up 30-40%.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Run the modified test suites

Closes #29176 from MaxGekk/jdbc-suite-before-all.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
Max Gekk 2020-07-22 08:32:01 +00:00 committed by Wenchen Fan
parent 29b7eaa438
commit feca9edbdd
2 changed files with 8 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
View file

@ -26,7 +26,8 @@ import org.apache.spark.util.Utils
class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkContext {
before {
override def beforeAll(): Unit = {
super.beforeAll()
Utils.classForName("org.apache.derby.jdbc.EmbeddedDriver")
val conn = DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb;create=true")
try {
@ -96,7 +97,7 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
assert(rdd.reduce(_ + _) === 5050)
}
after {
override def afterAll(): Unit = {
try {
DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb;shutdown=true")
} catch {
@ -104,5 +105,6 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
// Normal single database shutdown
// https://db.apache.org/derby/docs/10.2/ref/rrefexcept71493.html
}
super.afterAll()
}
}

6
sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
View file

@ -71,7 +71,8 @@ class JDBCSuite extends QueryTest
}
}
before {
override def beforeAll(): Unit = {
super.beforeAll()
Utils.classForName("org.h2.Driver")
// Extra properties that will be specified for our database. We need these to test
// usage of parameters from OPTIONS clause in queries.
@ -273,8 +274,9 @@ class JDBCSuite extends QueryTest
// Untested: IDENTITY, OTHER, UUID, ARRAY, and GEOMETRY types.
}
after {
override def afterAll(): Unit = {
conn.close()
super.afterAll()
}
// Check whether the tables are fetched in the expected degree of parallelism