[SPARK-7502] DAG visualization: gracefully handle removed stages
Old stages are removed without much feedback to the user. This happens very often in streaming. See screenshots below for more detail. zsxwing **Before** <img src="https://cloud.githubusercontent.com/assets/2133137/7621031/643cc1e0-f978-11e4-8f42-09decaac44a7.png" width="500px"/> ------------------------- **After** <img src="https://cloud.githubusercontent.com/assets/2133137/7621037/6e37348c-f978-11e4-84a5-e44e154f9b13.png" width="400px"/> Author: Andrew Or <andrew@databricks.com> Closes #6132 from andrewor14/dag-viz-remove-gracefully and squashes the following commits: 43175cd [Andrew Or] Handle removed jobs and stages gracefully
This commit is contained in:
parent
44403414d3
commit
aa1837875a
|
@ -44,6 +44,10 @@
|
|||
stroke-width: 1px;
|
||||
}
|
||||
|
||||
#dag-viz-graph div#empty-dag-viz-message {
|
||||
margin: 15px;
|
||||
}
|
||||
|
||||
/* Job page specific styles */
|
||||
|
||||
#dag-viz-graph svg.job marker#marker-arrow path {
|
||||
|
|
|
@ -86,7 +86,7 @@ function toggleDagViz(forJob) {
|
|||
$(arrowSelector).toggleClass('arrow-open');
|
||||
var shouldShow = $(arrowSelector).hasClass("arrow-open");
|
||||
if (shouldShow) {
|
||||
var shouldRender = graphContainer().select("svg").empty();
|
||||
var shouldRender = graphContainer().select("*").empty();
|
||||
if (shouldRender) {
|
||||
renderDagViz(forJob);
|
||||
}
|
||||
|
@ -117,10 +117,18 @@ function renderDagViz(forJob) {
|
|||
|
||||
// If there is not a dot file to render, fail fast and report error
|
||||
var jobOrStage = forJob ? "job" : "stage";
|
||||
if (metadataContainer().empty()) {
|
||||
graphContainer()
|
||||
.append("div")
|
||||
.text("No visualization information available for this " + jobOrStage);
|
||||
if (metadataContainer().empty() ||
|
||||
metadataContainer().selectAll("div").empty()) {
|
||||
var message =
|
||||
"<b>No visualization information available for this " + jobOrStage + "!</b><br/>" +
|
||||
"If this is an old " + jobOrStage + ", its visualization metadata may have been " +
|
||||
"cleaned up over time.<br/> You may consider increasing the value of ";
|
||||
if (forJob) {
|
||||
message += "<i>spark.ui.retainedJobs</i> and <i>spark.ui.retainedStages</i>.";
|
||||
} else {
|
||||
message += "<i>spark.ui.retainedStages</i>";
|
||||
}
|
||||
graphContainer().append("div").attr("id", "empty-dag-viz-message").html(message);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
@ -42,9 +42,14 @@ private[ui] class RDDOperationGraphListener(conf: SparkConf) extends SparkListen
|
|||
|
||||
/** Return the graph metadata for the given stage, or None if no such information exists. */
|
||||
def getOperationGraphForJob(jobId: Int): Seq[RDDOperationGraph] = {
|
||||
jobIdToStageIds.get(jobId)
|
||||
.map { sids => sids.flatMap { sid => stageIdToGraph.get(sid) } }
|
||||
.getOrElse { Seq.empty }
|
||||
val stageIds = jobIdToStageIds.get(jobId).getOrElse { Seq.empty }
|
||||
val graphs = stageIds.flatMap { sid => stageIdToGraph.get(sid) }
|
||||
// If the metadata for some stages have been removed, do not bother rendering this job
|
||||
if (stageIds.size != graphs.size) {
|
||||
Seq.empty
|
||||
} else {
|
||||
graphs
|
||||
}
|
||||
}
|
||||
|
||||
/** Return the graph metadata for the given stage, or None if no such information exists. */
|
||||
|
|
Loading…
Reference in a new issue