ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-32564][SQL][TEST][FOLLOWUP] Re-enable TPCDSQuerySuite with empty tables

Browse source 
### What changes were proposed in this pull request?

This is the follow-up PR of #29384 to address the cloud-fan comment: https://github.com/apache/spark/pull/29384#issuecomment-670595111
This PR re-enables `TPCDSQuerySuite` with empty tables for better test coverages.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #29391 from maropu/SPARK-32564-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
This commit is contained in:
Takeshi Yamamuro 2020-08-08 16:33:25 -07:00 committed by Dongjoon Hyun
parent 8062c1f777
commit 1df855bef2
2 changed files with 22 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
View file

@ -17,6 +17,7 @@
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.util.resourceToString
import org.apache.spark.sql.internal.SQLConf
@ -46,11 +47,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
"q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
"q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
val sqlConfgs = Seq(
SQLConf.CBO_ENABLED.key -> "true",
SQLConf.PLAN_STATS_ENABLED.key -> "true",
SQLConf.JOIN_REORDER_ENABLED.key -> "true"
)
val sqlConfgs: Seq[(String, String)] = Nil
tpcdsQueries.foreach { name =>
val queryString = resourceToString(s"tpcds/$name.sql",
@ -104,3 +101,23 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
}
}
}
class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
override def beforeAll(): Unit = {
super.beforeAll()
for (tableName <- tableNames) {
// To simulate plan generation on actual TPCDS data, injects data stats here
spark.sessionState.catalog.alterTableStats(
TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
}
}
// Sets configurations for enabling the optimization rules that
// exploit data statistics.
override val sqlConfgs = Seq(
SQLConf.CBO_ENABLED.key -> "true",
SQLConf.PLAN_STATS_ENABLED.key -> "true",
SQLConf.JOIN_REORDER_ENABLED.key -> "true"
)
}

6
sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
View file

@ -17,8 +17,6 @@
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.TableIdentifier
trait TPCDSSchema {
private val tableColumns = Map(
@ -257,9 +255,5 @@ trait TPCDSSchema {
|USING $format
|${options.mkString("\n")}
""".stripMargin)
// To simulate plan generation on actual TPCDS data, injects data stats here
spark.sessionState.catalog.alterTableStats(
TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
}
}