ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-25626][SQL][TEST] Improve the test execution time of HiveClientSuites

Browse source 
## What changes were proposed in this pull request?
Improve the runtime by reducing the number of partitions created in the test. The number of partitions are reduced from 280 to 60.

Here are the test times for the `getPartitionsByFilter returns all partitions` test  on my laptop.

```
[info] - 0.13: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (4 seconds, 230 milliseconds)
[info] - 0.14: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (3 seconds, 576 milliseconds)
[info] - 1.0: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (3 seconds, 495 milliseconds)
[info] - 1.1: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (6 seconds, 728 milliseconds)
[info] - 1.2: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (7 seconds, 260 milliseconds)
[info] - 2.0: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (8 seconds, 270 milliseconds)
[info] - 2.1: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (6 seconds, 856 milliseconds)
[info] - 2.2: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (7 seconds, 587 milliseconds)
[info] - 2.3: getPartitionsByFilter returns all partitions when hive.metastore.try.direct.sql=false (7 seconds, 230 milliseconds)
## How was this patch tested?
Test only.

Closes #22644 from dilipbiswal/SPARK-25626.

Authored-by: Dilip Biswal <dbiswal@us.ibm.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
This commit is contained in:
Dilip Biswal 2018-10-05 14:39:30 -07:00 committed by gatorsmile
parent 7dcc90fbb8
commit a433fbcee6
1 changed files with 35 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
View file

@ -32,7 +32,7 @@ class HiveClientSuite(version: String)
private val tryDirectSqlKey = HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL.varname
private val testPartitionCount = 3 * 24 * 4
private val testPartitionCount = 3 * 5 * 4
private def init(tryDirectSql: Boolean): HiveClient = {
val storageFormat = CatalogStorageFormat(
@ -51,7 +51,7 @@ class HiveClientSuite(version: String)
val partitions =
for {
ds <- 20170101 to 20170103
h <- 0 to 23
h <- 0 to 4
chunk <- Seq("aa", "ab", "ba", "bb")
} yield CatalogTablePartition(Map(
"ds" -> ds.toString,
@ -92,7 +92,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds") <=> 20170101,
20170101 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -100,7 +100,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds") === 20170101,
20170101 to 20170101,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -118,7 +118,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("chunk") === "aa",
20170101 to 20170103,
0 to 23,
0 to 4,
"aa" :: Nil)
}
@ -126,7 +126,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("chunk").cast(IntegerType) === 1,
20170101 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -134,7 +134,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("chunk").cast(BooleanType) === true,
20170101 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -142,23 +142,23 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
Literal(20170101) === attr("ds"),
20170101 to 20170101,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
test("getPartitionsByFilter: ds=20170101 and h=10") {
test("getPartitionsByFilter: ds=20170101 and h=2") {
testMetastorePartitionFiltering(
attr("ds") === 20170101 && attr("h") === 10,
attr("ds") === 20170101 && attr("h") === 2,
20170101 to 20170101,
10 to 10,
2 to 2,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
test("getPartitionsByFilter: cast(ds as long)=20170101L and h=10") {
test("getPartitionsByFilter: cast(ds as long)=20170101L and h=2") {
testMetastorePartitionFiltering(
attr("ds").cast(LongType) === 20170101L && attr("h") === 10,
attr("ds").cast(LongType) === 20170101L && attr("h") === 2,
20170101 to 20170101,
10 to 10,
2 to 2,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -166,7 +166,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds") === 20170101 || attr("ds") === 20170102,
20170101 to 20170102,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -174,7 +174,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds").in(20170102, 20170103),
20170102 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -182,7 +182,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds").cast(LongType).in(20170102L, 20170103L),
20170102 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
@ -190,7 +190,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds").in(20170102, 20170103),
20170102 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil, {
case expr @ In(v, list) if expr.inSetConvertible =>
InSet(v, list.map(_.eval(EmptyRow)).toSet)
@ -202,7 +202,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("ds").cast(LongType).in(20170102L, 20170103L),
20170102 to 20170103,
0 to 23,
0 to 4,
"aa" :: "ab" :: "ba" :: "bb" :: Nil, {
case expr @ In(v, list) if expr.inSetConvertible =>
InSet(v, list.map(_.eval(EmptyRow)).toSet)
@ -213,7 +213,7 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("chunk").in("ab", "ba"),
20170101 to 20170103,
0 to 23,
0 to 4,
"ab" :: "ba" :: Nil)
}
@ -221,34 +221,34 @@ class HiveClientSuite(version: String)
testMetastorePartitionFiltering(
attr("chunk").in("ab", "ba"),
20170101 to 20170103,
0 to 23,
0 to 4,
"ab" :: "ba" :: Nil, {
case expr @ In(v, list) if expr.inSetConvertible =>
InSet(v, list.map(_.eval(EmptyRow)).toSet)
})
}
test("getPartitionsByFilter: (ds=20170101 and h>=8) or (ds=20170102 and h<8)") {
val day1 = (20170101 to 20170101, 8 to 23, Seq("aa", "ab", "ba", "bb"))
val day2 = (20170102 to 20170102, 0 to 7, Seq("aa", "ab", "ba", "bb"))
testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 8) ||
(attr("ds") === 20170102 && attr("h") < 8), day1 :: day2 :: Nil)
test("getPartitionsByFilter: (ds=20170101 and h>=2) or (ds=20170102 and h<2)") {
val day1 = (20170101 to 20170101, 2 to 4, Seq("aa", "ab", "ba", "bb"))
val day2 = (20170102 to 20170102, 0 to 1, Seq("aa", "ab", "ba", "bb"))
testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 2) ||
(attr("ds") === 20170102 && attr("h") < 2), day1 :: day2 :: Nil)
}
test("getPartitionsByFilter: (ds=20170101 and h>=8) or (ds=20170102 and h<(7+1))") {
val day1 = (20170101 to 20170101, 8 to 23, Seq("aa", "ab", "ba", "bb"))
test("getPartitionsByFilter: (ds=20170101 and h>=2) or (ds=20170102 and h<(1+1))") {
val day1 = (20170101 to 20170101, 2 to 4, Seq("aa", "ab", "ba", "bb"))
// Day 2 should include all hours because we can't build a filter for h<(7+1)
val day2 = (20170102 to 20170102, 0 to 23, Seq("aa", "ab", "ba", "bb"))
testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 8) ||
(attr("ds") === 20170102 && attr("h") < (Literal(7) + 1)), day1 :: day2 :: Nil)
val day2 = (20170102 to 20170102, 0 to 4, Seq("aa", "ab", "ba", "bb"))
testMetastorePartitionFiltering((attr("ds") === 20170101 && attr("h") >= 2) ||
(attr("ds") === 20170102 && attr("h") < (Literal(1) + 1)), day1 :: day2 :: Nil)
}
test("getPartitionsByFilter: " +
"chunk in ('ab', 'ba') and ((ds=20170101 and h>=8) or (ds=20170102 and h<8))") {
val day1 = (20170101 to 20170101, 8 to 23, Seq("ab", "ba"))
val day2 = (20170102 to 20170102, 0 to 7, Seq("ab", "ba"))
"chunk in ('ab', 'ba') and ((ds=20170101 and h>=2) or (ds=20170102 and h<2))") {
val day1 = (20170101 to 20170101, 2 to 4, Seq("ab", "ba"))
val day2 = (20170102 to 20170102, 0 to 1, Seq("ab", "ba"))
testMetastorePartitionFiltering(attr("chunk").in("ab", "ba") &&
((attr("ds") === 20170101 && attr("h") >= 8) || (attr("ds") === 20170102 && attr("h") < 8)),
((attr("ds") === 20170101 && attr("h") >= 2) || (attr("ds") === 20170102 && attr("h") < 2)),
day1 :: day2 :: Nil)
}