[SPARK-27045][SQL] SQL tab in UI shows actual SQL instead of callsite in case of SparkSQLDriver
## What changes were proposed in this pull request? When we run sql in spark via SparkSQLDriver (thrift server, spark-sql), SQL string is siet via ``setJobDescription``. the SparkUI SQL tab must show SQL instead of stacktrace in case ``setJobDescription`` is set which is more useful to end user. Instead it currently shows in description column the callsite shortform which is less useful ![image](https://user-images.githubusercontent.com/22072336/53734682-aaa7d900-3eaa-11e9-957b-0e5006db417e.png) ## How was this patch tested? Manually: ![image](https://user-images.githubusercontent.com/22072336/53734657-9f54ad80-3eaa-11e9-8dc5-2b38f6970f4e.png) Closes #23958 from ajithme/sqlui. Authored-by: Ajith <ajith2489@gmail.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
parent
dccf6615c3
commit
e60d8fce0b
|
@ -132,4 +132,10 @@ object StaticSQLConf {
|
||||||
.intConf
|
.intConf
|
||||||
.createWithDefault(1000)
|
.createWithDefault(1000)
|
||||||
|
|
||||||
|
val SQL_EVENT_TRUNCATE_LENGTH = buildStaticConf("spark.sql.event.truncate.length")
|
||||||
|
.doc("Threshold of SQL length beyond which it will be truncated before adding to " +
|
||||||
|
"event. Defaults to no truncation. If set to 0, callsite will be logged instead.")
|
||||||
|
.intConf
|
||||||
|
.checkValue(_ >= 0, "Must be set greater or equal to zero")
|
||||||
|
.createWithDefault(Int.MaxValue)
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,9 +20,12 @@ package org.apache.spark.sql.execution
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
import org.apache.spark.SparkContext
|
||||||
import org.apache.spark.internal.config.Tests.IS_TESTING
|
import org.apache.spark.internal.config.Tests.IS_TESTING
|
||||||
import org.apache.spark.sql.SparkSession
|
import org.apache.spark.sql.SparkSession
|
||||||
import org.apache.spark.sql.execution.ui.{SparkListenerSQLExecutionEnd, SparkListenerSQLExecutionStart}
|
import org.apache.spark.sql.execution.ui.{SparkListenerSQLExecutionEnd, SparkListenerSQLExecutionStart}
|
||||||
|
import org.apache.spark.sql.internal.StaticSQLConf.SQL_EVENT_TRUNCATE_LENGTH
|
||||||
|
import org.apache.spark.util.Utils
|
||||||
|
|
||||||
object SQLExecution {
|
object SQLExecution {
|
||||||
|
|
||||||
|
@ -71,13 +74,23 @@ object SQLExecution {
|
||||||
// streaming queries would give us call site like "run at <unknown>:0"
|
// streaming queries would give us call site like "run at <unknown>:0"
|
||||||
val callSite = sc.getCallSite()
|
val callSite = sc.getCallSite()
|
||||||
|
|
||||||
|
val truncateLength = sc.conf.get(SQL_EVENT_TRUNCATE_LENGTH)
|
||||||
|
|
||||||
|
val desc = Option(sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION))
|
||||||
|
.filter(_ => truncateLength > 0)
|
||||||
|
.map { sqlStr =>
|
||||||
|
val redactedStr = Utils
|
||||||
|
.redact(sparkSession.sessionState.conf.stringRedactionPattern, sqlStr)
|
||||||
|
redactedStr.substring(0, Math.min(truncateLength, redactedStr.length))
|
||||||
|
}.getOrElse(callSite.shortForm)
|
||||||
|
|
||||||
withSQLConfPropagated(sparkSession) {
|
withSQLConfPropagated(sparkSession) {
|
||||||
var ex: Option[Exception] = None
|
var ex: Option[Exception] = None
|
||||||
val startTime = System.nanoTime()
|
val startTime = System.nanoTime()
|
||||||
try {
|
try {
|
||||||
sc.listenerBus.post(SparkListenerSQLExecutionStart(
|
sc.listenerBus.post(SparkListenerSQLExecutionStart(
|
||||||
executionId = executionId,
|
executionId = executionId,
|
||||||
description = callSite.shortForm,
|
description = desc,
|
||||||
details = callSite.longForm,
|
details = callSite.longForm,
|
||||||
physicalPlanDescription = queryExecution.toString,
|
physicalPlanDescription = queryExecution.toString,
|
||||||
// `queryExecution.executedPlan` triggers query planning. If it fails, the exception
|
// `queryExecution.executedPlan` triggers query planning. If it fails, the exception
|
||||||
|
|
|
@ -388,14 +388,15 @@ private[ui] class ExecutionPagedTable(
|
||||||
+details
|
+details
|
||||||
</span> ++
|
</span> ++
|
||||||
<div class="stage-details collapsed">
|
<div class="stage-details collapsed">
|
||||||
<pre>{execution.details}</pre>
|
<pre>{execution.description}<br></br>{execution.details}</pre>
|
||||||
</div>
|
</div>
|
||||||
} else {
|
} else {
|
||||||
Nil
|
Nil
|
||||||
}
|
}
|
||||||
|
|
||||||
val desc = if (execution.description != null && execution.description.nonEmpty) {
|
val desc = if (execution.description != null && execution.description.nonEmpty) {
|
||||||
<a href={executionURL(execution.executionId)}>{execution.description}</a>
|
<a href={executionURL(execution.executionId)} class="description-input">
|
||||||
|
{execution.description}</a>
|
||||||
} else {
|
} else {
|
||||||
<a href={executionURL(execution.executionId)}>{execution.executionId}</a>
|
<a href={executionURL(execution.executionId)}>{execution.executionId}</a>
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue