[SPARK-7371] [SPARK-7377] [SPARK-7408] DAG visualization addendum (#5729)
This is a follow-up patch for #5729.
**[SPARK-7408]** Move as much style code from JS to CSS as possible
**[SPARK-7377]** Fix JS error if a job / stage contains only one RDD
**[SPARK-7371]** Decrease emphasis on RDD on stage page as requested by mateiz pwendell
This patch also includes general code clean up.
<img src="https://issues.apache.org/jira/secure/attachment/12730992/before-after.png" width="500px"></img>
Author: Andrew Or <andrew@databricks.com>
Closes #5954 from andrewor14/viz-emphasize-rdd and squashes the following commits:
3c0d4f0 [Andrew Or] Guard against JS error by rendering arrows only if needed
f23e15b [Andrew Or] Merge branch 'master' of github.com:apache/spark into viz-emphasize-rdd
565801f [Andrew Or] Clean up code
9dab5f0 [Andrew Or] Move styling from JS to CSS + clean up code
107c0b6 [Andrew Or] Tweak background color, stroke width, font size etc.
1610c62 [Andrew Or] Implement cluster padding for stage page
(cherry picked from commit 8fa6829f5e
)
Signed-off-by: Andrew Or <andrew@databricks.com>
This commit is contained in:
parent
ba24dfae72
commit
76e8344f20
File diff suppressed because one or more lines are too long
|
@ -0,0 +1,83 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#dag-viz-graph svg path {
|
||||||
|
stroke: #444444;
|
||||||
|
stroke-width: 1.5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg g.cluster rect {
|
||||||
|
stroke-width: 4px;
|
||||||
|
stroke-opacity: 0.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg g.node circle,
|
||||||
|
#dag-viz-graph svg g.node rect {
|
||||||
|
fill: #444444;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg g.node.cached circle,
|
||||||
|
#dag-viz-graph svg g.node.cached rect {
|
||||||
|
fill: #FF0000;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Job page specific styles */
|
||||||
|
|
||||||
|
#dag-viz-graph svg.job marker#marker-arrow path {
|
||||||
|
fill: #444444;
|
||||||
|
stroke-width: 0px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.job g.cluster rect {
|
||||||
|
fill: #FFFFFF;
|
||||||
|
stroke: #AADFFF;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.job g.cluster[id*="stage"] rect {
|
||||||
|
stroke: #FFDDEE;
|
||||||
|
stroke-width: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.job g#cross-stage-edges path {
|
||||||
|
fill: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.job g.cluster text {
|
||||||
|
fill: #AAAAAA;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Stage page specific styles */
|
||||||
|
|
||||||
|
#dag-viz-graph svg.stage g.cluster rect {
|
||||||
|
fill: #F0F8FF;
|
||||||
|
stroke: #AADFFF;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.stage g.cluster[id*="stage"] rect {
|
||||||
|
fill: #FFFFFF;
|
||||||
|
stroke: #FFDDEE;
|
||||||
|
stroke-width: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.stage g.node g.label text tspan {
|
||||||
|
fill: #FFFFFF;
|
||||||
|
}
|
||||||
|
|
||||||
|
#dag-viz-graph svg.stage g.cluster text {
|
||||||
|
fill: #444444;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
|
@ -23,10 +23,10 @@
|
||||||
* (2) an RDD and its operation scopes, and
|
* (2) an RDD and its operation scopes, and
|
||||||
* (3) an RDD's operation scopes and the stage / job hierarchy
|
* (3) an RDD's operation scopes and the stage / job hierarchy
|
||||||
*
|
*
|
||||||
* An operation scope is a general, named code block representing an operation
|
* An operation scope is a general, named code block that instantiates RDDs
|
||||||
* that instantiates RDDs (e.g. filter, textFile, reduceByKey). An operation
|
* (e.g. filter, textFile, reduceByKey). An operation scope can be nested inside
|
||||||
* scope can be nested inside of other scopes if the corresponding RDD operation
|
* of other scopes if the corresponding RDD operation invokes other such operations
|
||||||
* invokes other such operations (for more detail, see o.a.s.rdd.operationScope).
|
* (for more detail, see o.a.s.rdd.RDDOperationScope).
|
||||||
*
|
*
|
||||||
* A stage may include one or more operation scopes if the RDD operations are
|
* A stage may include one or more operation scopes if the RDD operations are
|
||||||
* streamlined into one stage (e.g. rdd.map(...).filter(...).flatMap(...)).
|
* streamlined into one stage (e.g. rdd.map(...).filter(...).flatMap(...)).
|
||||||
|
@ -52,14 +52,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
var VizConstants = {
|
var VizConstants = {
|
||||||
rddColor: "#444444",
|
svgMarginX: 20,
|
||||||
rddCachedColor: "#FF0000",
|
|
||||||
rddOperationColor: "#AADFFF",
|
|
||||||
stageColor: "#FFDDEE",
|
|
||||||
clusterLabelColor: "#888888",
|
|
||||||
edgeColor: "#444444",
|
|
||||||
edgeWidth: "1.5px",
|
|
||||||
svgMarginX: 0,
|
|
||||||
svgMarginY: 20,
|
svgMarginY: 20,
|
||||||
stageSep: 50,
|
stageSep: 50,
|
||||||
graphPrefix: "graph_",
|
graphPrefix: "graph_",
|
||||||
|
@ -69,13 +62,21 @@ var VizConstants = {
|
||||||
stageClusterPrefix: "cluster_stage_"
|
stageClusterPrefix: "cluster_stage_"
|
||||||
};
|
};
|
||||||
|
|
||||||
// Helper d3 accessors for the elements that contain our graph and its metadata
|
var JobPageVizConstants = {
|
||||||
function graphContainer() { return d3.select("#dag-viz-graph"); }
|
clusterLabelSize: 11,
|
||||||
function metadataContainer() { return d3.select("#dag-viz-metadata"); }
|
stageClusterLabelSize: 14
|
||||||
|
}
|
||||||
|
|
||||||
|
var StagePageVizConstants = {
|
||||||
|
clusterLabelSize: 14,
|
||||||
|
stageClusterLabelSize: 18
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Show or hide the RDD DAG visualization.
|
* Show or hide the RDD DAG visualization.
|
||||||
|
*
|
||||||
* The graph is only rendered the first time this is called.
|
* The graph is only rendered the first time this is called.
|
||||||
|
* This is the narrow interface called from the Scala UI code.
|
||||||
*/
|
*/
|
||||||
function toggleDagViz(forJob) {
|
function toggleDagViz(forJob) {
|
||||||
var arrowSelector = ".expand-dag-viz-arrow";
|
var arrowSelector = ".expand-dag-viz-arrow";
|
||||||
|
@ -113,70 +114,52 @@ function toggleDagViz(forJob) {
|
||||||
function renderDagViz(forJob) {
|
function renderDagViz(forJob) {
|
||||||
|
|
||||||
// If there is not a dot file to render, fail fast and report error
|
// If there is not a dot file to render, fail fast and report error
|
||||||
|
var jobOrStage = forJob ? "job" : "stage";
|
||||||
if (metadataContainer().empty()) {
|
if (metadataContainer().empty()) {
|
||||||
graphContainer().append("div").text(
|
graphContainer()
|
||||||
"No visualization information available for this " + (forJob ? "job" : "stage"));
|
.append("div")
|
||||||
|
.text("No visualization information available for this " + jobOrStage);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var svg = graphContainer().append("svg");
|
// Render
|
||||||
|
var svg = graphContainer()
|
||||||
|
.append("svg")
|
||||||
|
.attr("class", jobOrStage);
|
||||||
if (forJob) {
|
if (forJob) {
|
||||||
renderDagVizForJob(svg);
|
renderDagVizForJob(svg);
|
||||||
} else {
|
} else {
|
||||||
renderDagVizForStage(svg);
|
renderDagVizForStage(svg);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find cached RDDs
|
// Find cached RDDs and mark them as such
|
||||||
metadataContainer().selectAll(".cached-rdd").each(function(v) {
|
metadataContainer().selectAll(".cached-rdd").each(function(v) {
|
||||||
var nodeId = VizConstants.nodePrefix + d3.select(this).text();
|
var nodeId = VizConstants.nodePrefix + d3.select(this).text();
|
||||||
graphContainer().selectAll("#" + nodeId).classed("cached", true);
|
svg.selectAll("#" + nodeId).classed("cached", true);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Set the appropriate SVG dimensions to ensure that all elements are displayed
|
// More post-processing
|
||||||
var boundingBox = svg.node().getBBox();
|
drawClusterLabels(svg, forJob);
|
||||||
svg.style("width", (boundingBox.width + VizConstants.svgMarginX) + "px");
|
resizeSvg(svg);
|
||||||
svg.style("height", (boundingBox.height + VizConstants.svgMarginY) + "px");
|
|
||||||
|
|
||||||
// Add labels to clusters because dagre-d3 doesn't do this for us
|
|
||||||
svg.selectAll("g.cluster rect").each(function() {
|
|
||||||
var rect = d3.select(this);
|
|
||||||
var cluster = d3.select(this.parentNode);
|
|
||||||
// Shift the boxes up a little to make room for the labels
|
|
||||||
rect.attr("y", toFloat(rect.attr("y")) - 10);
|
|
||||||
rect.attr("height", toFloat(rect.attr("height")) + 10);
|
|
||||||
var labelX = toFloat(rect.attr("x")) + toFloat(rect.attr("width")) - 5;
|
|
||||||
var labelY = toFloat(rect.attr("y")) + 15;
|
|
||||||
var labelText = cluster.attr("name").replace(VizConstants.clusterPrefix, "");
|
|
||||||
cluster.append("text")
|
|
||||||
.attr("x", labelX)
|
|
||||||
.attr("y", labelY)
|
|
||||||
.attr("text-anchor", "end")
|
|
||||||
.text(labelText);
|
|
||||||
});
|
|
||||||
|
|
||||||
// We have shifted a few elements upwards, so we should fix the SVG views
|
|
||||||
var startX = -VizConstants.svgMarginX;
|
|
||||||
var startY = -VizConstants.svgMarginY;
|
|
||||||
var endX = toFloat(svg.style("width")) + VizConstants.svgMarginX;
|
|
||||||
var endY = toFloat(svg.style("height")) + VizConstants.svgMarginY;
|
|
||||||
var newViewBox = startX + " " + startY + " " + endX + " " + endY;
|
|
||||||
svg.attr("viewBox", newViewBox);
|
|
||||||
|
|
||||||
// Lastly, apply some custom style to the DAG
|
|
||||||
styleDagViz(forJob);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Render the RDD DAG visualization for a stage. */
|
/* Render the RDD DAG visualization on the stage page. */
|
||||||
function renderDagVizForStage(svgContainer) {
|
function renderDagVizForStage(svgContainer) {
|
||||||
var metadata = metadataContainer().select(".stage-metadata");
|
var metadata = metadataContainer().select(".stage-metadata");
|
||||||
var dot = metadata.select(".dot-file").text();
|
var dot = metadata.select(".dot-file").text();
|
||||||
var containerId = VizConstants.graphPrefix + metadata.attr("stageId");
|
var containerId = VizConstants.graphPrefix + metadata.attr("stage-id");
|
||||||
var container = svgContainer.append("g").attr("id", containerId);
|
var container = svgContainer.append("g").attr("id", containerId);
|
||||||
renderDot(dot, container);
|
renderDot(dot, container);
|
||||||
|
|
||||||
|
// Round corners on RDDs
|
||||||
|
svgContainer
|
||||||
|
.selectAll("g.node rect")
|
||||||
|
.attr("rx", "5")
|
||||||
|
.attr("ry", "5");
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Render the RDD DAG visualization for a job.
|
* Render the RDD DAG visualization on the job page.
|
||||||
*
|
*
|
||||||
* Due to limitations in dagre-d3, each stage is rendered independently so that
|
* Due to limitations in dagre-d3, each stage is rendered independently so that
|
||||||
* we have more control on how to position them. Unfortunately, this means we
|
* we have more control on how to position them. Unfortunately, this means we
|
||||||
|
@ -186,32 +169,46 @@ function renderDagVizForStage(svgContainer) {
|
||||||
function renderDagVizForJob(svgContainer) {
|
function renderDagVizForJob(svgContainer) {
|
||||||
var crossStageEdges = [];
|
var crossStageEdges = [];
|
||||||
|
|
||||||
|
// Each div.stage-metadata contains the information needed to generate the graph
|
||||||
|
// for a stage. This includes the DOT file produced from the appropriate UI listener,
|
||||||
|
// any incoming and outgoing edges, and any cached RDDs that belong to this stage.
|
||||||
metadataContainer().selectAll(".stage-metadata").each(function(d, i) {
|
metadataContainer().selectAll(".stage-metadata").each(function(d, i) {
|
||||||
var metadata = d3.select(this);
|
var metadata = d3.select(this);
|
||||||
var dot = metadata.select(".dot-file").text();
|
var dot = metadata.select(".dot-file").text();
|
||||||
var stageId = metadata.attr("stageId");
|
var stageId = metadata.attr("stage-id");
|
||||||
var containerId = VizConstants.graphPrefix + stageId;
|
var containerId = VizConstants.graphPrefix + stageId;
|
||||||
// TODO: handle stage attempts
|
// Link each graph to the corresponding stage page (TODO: handle stage attempts)
|
||||||
var stageLink =
|
var stageLink =
|
||||||
"/stages/stage/?id=" + stageId.replace(VizConstants.stagePrefix, "") + "&attempt=0";
|
"/stages/stage/?id=" + stageId.replace(VizConstants.stagePrefix, "") + "&attempt=0";
|
||||||
var container = svgContainer
|
var container = svgContainer
|
||||||
.append("a").attr("xlink:href", stageLink)
|
.append("a")
|
||||||
.append("g").attr("id", containerId);
|
.attr("xlink:href", stageLink)
|
||||||
// Now we need to shift the container for this stage so it doesn't overlap
|
.append("g")
|
||||||
// with existing ones. We do not need to do this for the first stage.
|
.attr("id", containerId);
|
||||||
|
|
||||||
|
// Now we need to shift the container for this stage so it doesn't overlap with
|
||||||
|
// existing ones, taking into account the position and width of the last stage's
|
||||||
|
// container. We do not need to do this for the first stage of this job.
|
||||||
if (i > 0) {
|
if (i > 0) {
|
||||||
// Take into account the position and width of the last stage's container
|
var existingStages = svgContainer
|
||||||
var existingStages = stageClusters();
|
.selectAll("g.cluster")
|
||||||
|
.filter("[id*=\"" + VizConstants.stageClusterPrefix + "\"]");
|
||||||
if (!existingStages.empty()) {
|
if (!existingStages.empty()) {
|
||||||
var lastStage = existingStages[0].pop();
|
var lastStage = d3.select(existingStages[0].pop());
|
||||||
var lastStageId = d3.select(lastStage).attr("id");
|
var lastStageId = lastStage.attr("id");
|
||||||
var lastStageWidth = toFloat(d3.select("#" + lastStageId + " rect").attr("width"));
|
var lastStageWidth = toFloat(svgContainer
|
||||||
var lastStagePosition = getAbsolutePosition(lastStageId);
|
.select("#" + lastStageId)
|
||||||
|
.select("rect")
|
||||||
|
.attr("width"));
|
||||||
|
var lastStagePosition = getAbsolutePosition(lastStage);
|
||||||
var offset = lastStagePosition.x + lastStageWidth + VizConstants.stageSep;
|
var offset = lastStagePosition.x + lastStageWidth + VizConstants.stageSep;
|
||||||
container.attr("transform", "translate(" + offset + ", 0)");
|
container.attr("transform", "translate(" + offset + ", 0)");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Actually render the stage
|
||||||
renderDot(dot, container);
|
renderDot(dot, container);
|
||||||
|
|
||||||
// If there are any incoming edges into this graph, keep track of them to render
|
// If there are any incoming edges into this graph, keep track of them to render
|
||||||
// them separately later. Note that we cannot draw them now because we need to
|
// them separately later. Note that we cannot draw them now because we need to
|
||||||
// put these edges in a separate container that is on top of all stage graphs.
|
// put these edges in a separate container that is on top of all stage graphs.
|
||||||
|
@ -221,15 +218,7 @@ function renderDagVizForJob(svgContainer) {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// Draw edges that cross stages
|
drawCrossStageEdges(crossStageEdges, svgContainer);
|
||||||
if (crossStageEdges.length > 0) {
|
|
||||||
var container = svgContainer.append("g").attr("id", "cross-stage-edges");
|
|
||||||
for (var i = 0; i < crossStageEdges.length; i++) {
|
|
||||||
var fromRDDId = crossStageEdges[i][0];
|
|
||||||
var toRDDId = crossStageEdges[i][1];
|
|
||||||
connectRDDs(fromRDDId, toRDDId, container);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Render the dot file as an SVG in the given container. */
|
/* Render the dot file as an SVG in the given container. */
|
||||||
|
@ -243,99 +232,156 @@ function renderDot(dot, container) {
|
||||||
renderer(container, g);
|
renderer(container, g);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Style the visualization we just rendered. */
|
/* -------------------- *
|
||||||
function styleDagViz(forJob) {
|
* | Helper functions | *
|
||||||
graphContainer().selectAll("svg g.cluster rect")
|
* -------------------- */
|
||||||
.style("fill", "white")
|
|
||||||
.style("stroke", VizConstants.rddOperationColor)
|
|
||||||
.style("stroke-width", "4px")
|
|
||||||
.style("stroke-opacity", "0.5");
|
|
||||||
graphContainer().selectAll("svg g.cluster text")
|
|
||||||
.attr("fill", VizConstants.clusterLabelColor)
|
|
||||||
.attr("font-size", "11px");
|
|
||||||
graphContainer().selectAll("svg path")
|
|
||||||
.style("stroke", VizConstants.edgeColor)
|
|
||||||
.style("stroke-width", VizConstants.edgeWidth);
|
|
||||||
stageClusters()
|
|
||||||
.select("rect")
|
|
||||||
.style("stroke", VizConstants.stageColor)
|
|
||||||
.style("strokeWidth", "6px");
|
|
||||||
|
|
||||||
// Put an arrow at the end of every edge
|
// Helper d3 accessors
|
||||||
// We need to do this because we manually render some edges ourselves
|
function graphContainer() { return d3.select("#dag-viz-graph"); }
|
||||||
// For these edges, we borrow the arrow marker generated by dagre-d3
|
function metadataContainer() { return d3.select("#dag-viz-metadata"); }
|
||||||
var dagreD3Marker = graphContainer().select("svg g.edgePaths marker").node();
|
|
||||||
graphContainer().select("svg")
|
|
||||||
.append(function() { return dagreD3Marker.cloneNode(true); })
|
|
||||||
.attr("id", "marker-arrow")
|
|
||||||
.select("path")
|
|
||||||
.attr("fill", VizConstants.edgeColor)
|
|
||||||
.attr("strokeWidth", "0px");
|
|
||||||
graphContainer().selectAll("svg g > path").attr("marker-end", "url(#marker-arrow)");
|
|
||||||
graphContainer().selectAll("svg g.edgePaths def").remove(); // We no longer need these
|
|
||||||
|
|
||||||
// Apply any job or stage specific styles
|
/*
|
||||||
|
* Helper function to create draw a label for each cluster.
|
||||||
|
*
|
||||||
|
* We need to do this manually because dagre-d3 does not support labeling clusters.
|
||||||
|
* In general, the clustering support for dagre-d3 is quite limited at this point.
|
||||||
|
*/
|
||||||
|
function drawClusterLabels(svgContainer, forJob) {
|
||||||
if (forJob) {
|
if (forJob) {
|
||||||
styleDagVizForJob();
|
var clusterLabelSize = JobPageVizConstants.clusterLabelSize;
|
||||||
|
var stageClusterLabelSize = JobPageVizConstants.stageClusterLabelSize;
|
||||||
} else {
|
} else {
|
||||||
styleDagVizForStage();
|
var clusterLabelSize = StagePageVizConstants.clusterLabelSize;
|
||||||
|
var stageClusterLabelSize = StagePageVizConstants.stageClusterLabelSize;
|
||||||
}
|
}
|
||||||
}
|
svgContainer.selectAll("g.cluster").each(function() {
|
||||||
|
var cluster = d3.select(this);
|
||||||
/* Apply job-page-specific style to the visualization. */
|
var isStage = cluster.attr("id").indexOf(VizConstants.stageClusterPrefix) > -1;
|
||||||
function styleDagVizForJob() {
|
var labelSize = isStage ? stageClusterLabelSize : clusterLabelSize;
|
||||||
graphContainer().selectAll("svg g.node circle")
|
drawClusterLabel(cluster, labelSize);
|
||||||
.style("fill", VizConstants.rddColor);
|
});
|
||||||
// TODO: add a legend to explain what a highlighted dot means
|
|
||||||
graphContainer().selectAll("svg g.cached circle")
|
|
||||||
.style("fill", VizConstants.rddCachedColor);
|
|
||||||
graphContainer().selectAll("svg g#cross-stage-edges path")
|
|
||||||
.style("fill", "none");
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Apply stage-page-specific style to the visualization. */
|
|
||||||
function styleDagVizForStage() {
|
|
||||||
graphContainer().selectAll("svg g.node rect")
|
|
||||||
.style("fill", "none")
|
|
||||||
.style("stroke", VizConstants.rddColor)
|
|
||||||
.style("stroke-width", "2px")
|
|
||||||
.attr("rx", "5") // round corners
|
|
||||||
.attr("ry", "5");
|
|
||||||
// TODO: add a legend to explain what a highlighted RDD means
|
|
||||||
graphContainer().selectAll("svg g.cached rect")
|
|
||||||
.style("stroke", VizConstants.rddCachedColor);
|
|
||||||
graphContainer().selectAll("svg g.node g.label text tspan")
|
|
||||||
.style("fill", VizConstants.rddColor);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* (Job page only) Helper method to compute the absolute
|
* Helper function to draw a label for the given cluster element based on its name.
|
||||||
* position of the group element identified by the given ID.
|
*
|
||||||
|
* In the process, we need to expand the bounding box to make room for the label.
|
||||||
|
* We need to do this because dagre-d3 did not take this into account when it first
|
||||||
|
* rendered the bounding boxes. Note that this means we need to adjust the view box
|
||||||
|
* of the SVG afterwards since we shifted a few boxes around.
|
||||||
*/
|
*/
|
||||||
function getAbsolutePosition(groupId) {
|
function drawClusterLabel(d3cluster, fontSize) {
|
||||||
var obj = d3.select("#" + groupId).filter("g");
|
var cluster = d3cluster;
|
||||||
var _x = 0, _y = 0;
|
var rect = d3cluster.select("rect");
|
||||||
|
rect.attr("y", toFloat(rect.attr("y")) - fontSize);
|
||||||
|
rect.attr("height", toFloat(rect.attr("height")) + fontSize);
|
||||||
|
var labelX = toFloat(rect.attr("x")) + toFloat(rect.attr("width")) - fontSize / 2;
|
||||||
|
var labelY = toFloat(rect.attr("y")) + fontSize * 1.5;
|
||||||
|
var labelText = cluster.attr("name").replace(VizConstants.clusterPrefix, "");
|
||||||
|
cluster.append("text")
|
||||||
|
.attr("x", labelX)
|
||||||
|
.attr("y", labelY)
|
||||||
|
.attr("text-anchor", "end")
|
||||||
|
.style("font-size", fontSize)
|
||||||
|
.text(labelText);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Helper function to size the SVG appropriately such that all elements are displyed.
|
||||||
|
* This assumes that all outermost elements are clusters (rectangles).
|
||||||
|
*/
|
||||||
|
function resizeSvg(svg) {
|
||||||
|
var allClusters = svg.selectAll("g.cluster rect")[0];
|
||||||
|
var startX = -VizConstants.svgMarginX +
|
||||||
|
toFloat(d3.min(allClusters, function(e) {
|
||||||
|
return getAbsolutePosition(d3.select(e)).x;
|
||||||
|
}));
|
||||||
|
var startY = -VizConstants.svgMarginY +
|
||||||
|
toFloat(d3.min(allClusters, function(e) {
|
||||||
|
return getAbsolutePosition(d3.select(e)).y;
|
||||||
|
}));
|
||||||
|
var endX = VizConstants.svgMarginX +
|
||||||
|
toFloat(d3.max(allClusters, function(e) {
|
||||||
|
var t = d3.select(e)
|
||||||
|
return getAbsolutePosition(t).x + toFloat(t.attr("width"));
|
||||||
|
}));
|
||||||
|
var endY = VizConstants.svgMarginY +
|
||||||
|
toFloat(d3.max(allClusters, function(e) {
|
||||||
|
var t = d3.select(e)
|
||||||
|
return getAbsolutePosition(t).y + toFloat(t.attr("height"));
|
||||||
|
}));
|
||||||
|
var width = endX - startX;
|
||||||
|
var height = endY - startY;
|
||||||
|
svg.attr("viewBox", startX + " " + startY + " " + width + " " + height)
|
||||||
|
.attr("width", width)
|
||||||
|
.attr("height", height);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (Job page only) Helper function to draw edges that cross stage boundaries.
|
||||||
|
* We need to do this manually because we render each stage separately in dagre-d3.
|
||||||
|
*/
|
||||||
|
function drawCrossStageEdges(edges, svgContainer) {
|
||||||
|
if (edges.length == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Draw the paths first
|
||||||
|
var edgesContainer = svgContainer.append("g").attr("id", "cross-stage-edges");
|
||||||
|
for (var i = 0; i < edges.length; i++) {
|
||||||
|
var fromRDDId = edges[i][0];
|
||||||
|
var toRDDId = edges[i][1];
|
||||||
|
connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer);
|
||||||
|
}
|
||||||
|
// Now draw the arrows by borrowing the arrow marker generated by dagre-d3
|
||||||
|
var dagreD3Marker = svgContainer.select("g.edgePaths marker").node();
|
||||||
|
if (!dagreD3Marker.empty()) {
|
||||||
|
svgContainer
|
||||||
|
.append(function() { return dagreD3Marker.cloneNode(true); })
|
||||||
|
.attr("id", "marker-arrow")
|
||||||
|
svgContainer.selectAll("g > path").attr("marker-end", "url(#marker-arrow)");
|
||||||
|
svgContainer.selectAll("g.edgePaths def").remove(); // We no longer need these
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* (Job page only) Helper function to compute the absolute
|
||||||
|
* position of the specified element in our graph.
|
||||||
|
*/
|
||||||
|
function getAbsolutePosition(d3selection) {
|
||||||
|
if (d3selection.empty()) {
|
||||||
|
throw "Attempted to get absolute position of an empty selection.";
|
||||||
|
}
|
||||||
|
var obj = d3selection;
|
||||||
|
var _x = toFloat(obj.attr("x")) || 0;
|
||||||
|
var _y = toFloat(obj.attr("y")) || 0;
|
||||||
while (!obj.empty()) {
|
while (!obj.empty()) {
|
||||||
var transformText = obj.attr("transform");
|
var transformText = obj.attr("transform");
|
||||||
var translate = d3.transform(transformText).translate
|
if (transformText) {
|
||||||
_x += translate[0];
|
var translate = d3.transform(transformText).translate;
|
||||||
_y += translate[1];
|
_x += toFloat(translate[0]);
|
||||||
obj = d3.select(obj.node().parentNode).filter("g")
|
_y += toFloat(translate[1]);
|
||||||
|
}
|
||||||
|
// Climb upwards to find how our parents are translated
|
||||||
|
obj = d3.select(obj.node().parentNode);
|
||||||
|
// Stop when we've reached the graph container itself
|
||||||
|
if (obj.node() == graphContainer().node()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return { x: _x, y: _y };
|
return { x: _x, y: _y };
|
||||||
}
|
}
|
||||||
|
|
||||||
/* (Job page only) Connect two RDD nodes with a curved edge. */
|
/* (Job page only) Helper function to connect two RDDs with a curved edge. */
|
||||||
function connectRDDs(fromRDDId, toRDDId, container) {
|
function connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer) {
|
||||||
var fromNodeId = VizConstants.nodePrefix + fromRDDId;
|
var fromNodeId = VizConstants.nodePrefix + fromRDDId;
|
||||||
var toNodeId = VizConstants.nodePrefix + toRDDId
|
var toNodeId = VizConstants.nodePrefix + toRDDId;
|
||||||
var fromPos = getAbsolutePosition(fromNodeId);
|
var fromPos = getAbsolutePosition(svgContainer.select("#" + fromNodeId));
|
||||||
var toPos = getAbsolutePosition(toNodeId);
|
var toPos = getAbsolutePosition(svgContainer.select("#" + toNodeId));
|
||||||
|
|
||||||
// On the job page, RDDs are rendered as dots (circles). When rendering the path,
|
// On the job page, RDDs are rendered as dots (circles). When rendering the path,
|
||||||
// we need to account for the radii of these circles. Otherwise the arrow heads
|
// we need to account for the radii of these circles. Otherwise the arrow heads
|
||||||
// will bleed into the circle itself.
|
// will bleed into the circle itself.
|
||||||
var delta = toFloat(graphContainer()
|
var delta = toFloat(svgContainer
|
||||||
.select("g.node#" + toNodeId)
|
.select("g.node#" + toNodeId)
|
||||||
.select("circle")
|
.select("circle")
|
||||||
.attr("r"));
|
.attr("r"));
|
||||||
|
@ -375,18 +421,15 @@ function connectRDDs(fromRDDId, toRDDId, container) {
|
||||||
}
|
}
|
||||||
|
|
||||||
var line = d3.svg.line().interpolate("basis");
|
var line = d3.svg.line().interpolate("basis");
|
||||||
container.append("path").datum(points).attr("d", line);
|
edgesContainer.append("path").datum(points).attr("d", line);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Helper d3 accessor to clusters that represent stages. */
|
/* Helper function to convert attributes to numeric values. */
|
||||||
function stageClusters() {
|
|
||||||
return graphContainer().selectAll("g.cluster").filter(function() {
|
|
||||||
return d3.select(this).attr("id").indexOf(VizConstants.stageClusterPrefix) > -1;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Helper method to convert attributes to numeric values. */
|
|
||||||
function toFloat(f) {
|
function toFloat(f) {
|
||||||
return parseFloat(f.replace(/px$/, ""));
|
if (f) {
|
||||||
|
return parseFloat(f.toString().replace(/px$/, ""));
|
||||||
|
} else {
|
||||||
|
return f;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -156,13 +156,10 @@ private[spark] object UIUtils extends Logging {
|
||||||
|
|
||||||
def commonHeaderNodes: Seq[Node] = {
|
def commonHeaderNodes: Seq[Node] = {
|
||||||
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
|
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
|
||||||
<link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")}
|
<link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")} type="text/css" />
|
||||||
type="text/css" />
|
<link rel="stylesheet" href={prependBaseUri("/static/webui.css")} type="text/css" />
|
||||||
<link rel="stylesheet" href={prependBaseUri("/static/webui.css")}
|
<link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")} type="text/css" />
|
||||||
type="text/css" />
|
<link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")} type="text/css" />
|
||||||
<link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")}
|
|
||||||
typ="text/css" />
|
|
||||||
<link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")}></link>
|
|
||||||
<script src={prependBaseUri("/static/sorttable.js")} ></script>
|
<script src={prependBaseUri("/static/sorttable.js")} ></script>
|
||||||
<script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script>
|
<script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script>
|
||||||
<script src={prependBaseUri("/static/vis.min.js")}></script>
|
<script src={prependBaseUri("/static/vis.min.js")}></script>
|
||||||
|
@ -174,6 +171,7 @@ private[spark] object UIUtils extends Logging {
|
||||||
}
|
}
|
||||||
|
|
||||||
def vizHeaderNodes: Seq[Node] = {
|
def vizHeaderNodes: Seq[Node] = {
|
||||||
|
<link rel="stylesheet" href={prependBaseUri("/static/spark-dag-viz.css")} type="text/css" />
|
||||||
<script src={prependBaseUri("/static/d3.min.js")}></script>
|
<script src={prependBaseUri("/static/d3.min.js")}></script>
|
||||||
<script src={prependBaseUri("/static/dagre-d3.min.js")}></script>
|
<script src={prependBaseUri("/static/dagre-d3.min.js")}></script>
|
||||||
<script src={prependBaseUri("/static/graphlib-dot.min.js")}></script>
|
<script src={prependBaseUri("/static/graphlib-dot.min.js")}></script>
|
||||||
|
@ -358,7 +356,7 @@ private[spark] object UIUtils extends Logging {
|
||||||
<div id="dag-viz-metadata">
|
<div id="dag-viz-metadata">
|
||||||
{
|
{
|
||||||
graphs.map { g =>
|
graphs.map { g =>
|
||||||
<div class="stage-metadata" stageId={g.rootCluster.id} style="display:none">
|
<div class="stage-metadata" stage-id={g.rootCluster.id} style="display:none">
|
||||||
<div class="dot-file">{RDDOperationGraph.makeDotFile(g, forJob)}</div>
|
<div class="dot-file">{RDDOperationGraph.makeDotFile(g, forJob)}</div>
|
||||||
{ g.incomingEdges.map { e => <div class="incoming-edge">{e.fromId},{e.toId}</div> } }
|
{ g.incomingEdges.map { e => <div class="incoming-edge">{e.fromId},{e.toId}</div> } }
|
||||||
{ g.outgoingEdges.map { e => <div class="outgoing-edge">{e.fromId},{e.toId}</div> } }
|
{ g.outgoingEdges.map { e => <div class="outgoing-edge">{e.fromId},{e.toId}</div> } }
|
||||||
|
|
|
@ -181,22 +181,29 @@ private[ui] object RDDOperationGraph extends Logging {
|
||||||
if (forJob) {
|
if (forJob) {
|
||||||
s"""${node.id} [label=" " shape="circle" padding="5" labelStyle="font-size: 0"]"""
|
s"""${node.id} [label=" " shape="circle" padding="5" labelStyle="font-size: 0"]"""
|
||||||
} else {
|
} else {
|
||||||
s"""${node.id} [label="${node.name} (${node.id})"]"""
|
s"""${node.id} [label="${node.name} (${node.id})" padding="5" labelStyle="font-size: 10"]"""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Return the dot representation of a subgraph in an RDDOperationGraph. */
|
/** Return the dot representation of a subgraph in an RDDOperationGraph. */
|
||||||
private def makeDotSubgraph(
|
private def makeDotSubgraph(
|
||||||
scope: RDDOperationCluster,
|
cluster: RDDOperationCluster,
|
||||||
forJob: Boolean,
|
forJob: Boolean,
|
||||||
indent: String): String = {
|
indent: String): String = {
|
||||||
val subgraph = new StringBuilder
|
val subgraph = new StringBuilder
|
||||||
subgraph.append(indent + s"subgraph cluster${scope.id} {\n")
|
// TODO: move specific graph properties like these to spark-dag-viz.js
|
||||||
subgraph.append(indent + s""" label="${scope.name}";\n""")
|
val paddingTop = if (forJob) 10 else 20
|
||||||
scope.childNodes.foreach { node =>
|
subgraph.append(indent + s"subgraph cluster${cluster.id} {\n")
|
||||||
|
subgraph.append(indent + s""" label="${cluster.name}";\n""")
|
||||||
|
// If there are nested clusters, add some padding
|
||||||
|
// Do this for the stage page because we use bigger fonts there
|
||||||
|
if (cluster.childClusters.nonEmpty) {
|
||||||
|
subgraph.append(indent + s""" paddingTop="$paddingTop";\n""")
|
||||||
|
}
|
||||||
|
cluster.childNodes.foreach { node =>
|
||||||
subgraph.append(indent + s" ${makeDotNode(node, forJob)};\n")
|
subgraph.append(indent + s" ${makeDotNode(node, forJob)};\n")
|
||||||
}
|
}
|
||||||
scope.childClusters.foreach { cscope =>
|
cluster.childClusters.foreach { cscope =>
|
||||||
subgraph.append(makeDotSubgraph(cscope, forJob, indent + " "))
|
subgraph.append(makeDotSubgraph(cscope, forJob, indent + " "))
|
||||||
}
|
}
|
||||||
subgraph.append(indent + "}\n")
|
subgraph.append(indent + "}\n")
|
||||||
|
|
Loading…
Reference in a new issue