ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-17078][SQL][FOLLOWUP] Simplify explain cost command

Browse source 
## What changes were proposed in this pull request?

Usually when using explain cost command, users want to see the stats of plan. Since stats is only showed in optimized plan, it is more direct and convenient to include only optimized plan and physical plan in the output.

## How was this patch tested?

Enhanced existing test.

Author: Zhenhua Wang <wzh_zju@163.com>

Closes #18190 from wzhfy/simplifyExplainCost.
This commit is contained in:
Zhenhua Wang 2017-06-02 17:36:00 -07:00 committed by Wenchen Fan
parent 0eb1fc6cd5
commit 6de41e951f
3 changed files with 21 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
View file

@ -200,11 +200,7 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
""".stripMargin.trim
}
override def toString: String = completeString(appendStats = false)
def toStringWithStats: String = completeString(appendStats = true)
private def completeString(appendStats: Boolean): String = {
override def toString: String = {
def output = Utils.truncatedString(
analyzed.output.map(o => s"${o.name}: ${o.dataType.simpleString}"), ", ")
val analyzedPlan = Seq(
@ -212,25 +208,29 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
stringOrError(analyzed.treeString(verbose = true))
).filter(_.nonEmpty).mkString("\n")
val optimizedPlanString = if (appendStats) {
// trigger to compute stats for logical plans
optimizedPlan.stats(sparkSession.sessionState.conf)
optimizedPlan.treeString(verbose = true, addSuffix = true)
} else {
optimizedPlan.treeString(verbose = true)
}
s"""== Parsed Logical Plan ==
|${stringOrError(logical.treeString(verbose = true))}
|== Analyzed Logical Plan ==
|$analyzedPlan
|== Optimized Logical Plan ==
|${stringOrError(optimizedPlanString)}
|${stringOrError(optimizedPlan.treeString(verbose = true))}
|== Physical Plan ==
|${stringOrError(executedPlan.treeString(verbose = true))}
""".stripMargin.trim
}
def stringWithStats: String = {
// trigger to compute stats for logical plans
optimizedPlan.stats(sparkSession.sessionState.conf)
// only show optimized logical plan and physical plan
s"""== Optimized Logical Plan ==
|${stringOrError(optimizedPlan.treeString(verbose = true, addSuffix = true))}
|== Physical Plan ==
|${stringOrError(executedPlan.treeString(verbose = true))}
""".stripMargin.trim
}
/** A special namespace for commands that can be used to debug query execution. */
// scalastyle:off
object debug {

2
sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala
View file

@ -127,7 +127,7 @@ case class ExplainCommand(
} else if (extended) {
queryExecution.toString
} else if (cost) {
queryExecution.toStringWithStats
queryExecution.stringWithStats
} else {
queryExecution.simpleString
}

6
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
View file

@ -29,6 +29,12 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
import testImplicits._
test("show cost in explain command") {
// For readability, we only show optimized plan and physical plan in explain cost command
checkKeywordsExist(sql("EXPLAIN COST SELECT * FROM src "),
"Optimized Logical Plan", "Physical Plan")
checkKeywordsNotExist(sql("EXPLAIN COST SELECT * FROM src "),
"Parsed Logical Plan", "Analyzed Logical Plan")
// Only has sizeInBytes before ANALYZE command
checkKeywordsExist(sql("EXPLAIN COST SELECT * FROM src "), "sizeInBytes")
checkKeywordsNotExist(sql("EXPLAIN COST SELECT * FROM src "), "rowCount")