ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-7371] [SPARK-7377] [SPARK-7408] DAG visualization addendum (#5729)

Browse source 
This is a follow-up patch for #5729.

**[SPARK-7408]** Move as much style code from JS to CSS as possible
**[SPARK-7377]** Fix JS error if a job / stage contains only one RDD
**[SPARK-7371]** Decrease emphasis on RDD on stage page as requested by mateiz pwendell

This patch also includes general code clean up.

<img src="https://issues.apache.org/jira/secure/attachment/12730992/before-after.png" width="500px"></img>

Author: Andrew Or <andrew@databricks.com>

Closes #5954 from andrewor14/viz-emphasize-rdd and squashes the following commits:

3c0d4f0 [Andrew Or] Guard against JS error by rendering arrows only if needed
f23e15b [Andrew Or] Merge branch 'master' of github.com:apache/spark into viz-emphasize-rdd
565801f [Andrew Or] Clean up code
9dab5f0 [Andrew Or] Move styling from JS to CSS + clean up code
107c0b6 [Andrew Or] Tweak background color, stroke width, font size etc.
1610c62 [Andrew Or] Implement cluster padding for stage page

(cherry picked from commit 8fa6829f5e)
Signed-off-by: Andrew Or <andrew@databricks.com>
This commit is contained in:
Andrew Or 2015-05-06 17:52:34 -07:00
parent ba24dfae72
commit 76e8344f20
5 changed files with 312 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
core/src/main/resources/org/apache/spark/ui/static/dagre-d3.min.js vendored
View file

File diff suppressed because one or more lines are too long

83
core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css Normal file
View file

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#dag-viz-graph svg path {
stroke: #444444;
stroke-width: 1.5px;
}
#dag-viz-graph svg g.cluster rect {
stroke-width: 4px;
stroke-opacity: 0.5;
}
#dag-viz-graph svg g.node circle,
#dag-viz-graph svg g.node rect {
fill: #444444;
}
#dag-viz-graph svg g.node.cached circle,
#dag-viz-graph svg g.node.cached rect {
fill: #FF0000;
}
/* Job page specific styles */
#dag-viz-graph svg.job marker#marker-arrow path {
fill: #444444;
stroke-width: 0px;
}
#dag-viz-graph svg.job g.cluster rect {
fill: #FFFFFF;
stroke: #AADFFF;
}
#dag-viz-graph svg.job g.cluster[id*="stage"] rect {
stroke: #FFDDEE;
stroke-width: 6px;
}
#dag-viz-graph svg.job g#cross-stage-edges path {
fill: none;
}
#dag-viz-graph svg.job g.cluster text {
fill: #AAAAAA;
}
/* Stage page specific styles */
#dag-viz-graph svg.stage g.cluster rect {
fill: #F0F8FF;
stroke: #AADFFF;
}
#dag-viz-graph svg.stage g.cluster[id*="stage"] rect {
fill: #FFFFFF;
stroke: #FFDDEE;
stroke-width: 6px;
}
#dag-viz-graph svg.stage g.node g.label text tspan {
fill: #FFFFFF;
}
#dag-viz-graph svg.stage g.cluster text {
fill: #444444;
font-weight: bold;
}

361
core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
View file

@ -23,10 +23,10 @@
* (2) an RDD and its operation scopes, and * (2) an RDD and its operation scopes, and
* (3) an RDD's operation scopes and the stage / job hierarchy * (3) an RDD's operation scopes and the stage / job hierarchy
* *
* An operation scope is a general, named code block representing an operation * An operation scope is a general, named code block that instantiates RDDs
* that instantiates RDDs (e.g. filter, textFile, reduceByKey). An operation * (e.g. filter, textFile, reduceByKey). An operation scope can be nested inside
* scope can be nested inside of other scopes if the corresponding RDD operation * of other scopes if the corresponding RDD operation invokes other such operations
* invokes other such operations (for more detail, see o.a.s.rdd.operationScope). * (for more detail, see o.a.s.rdd.RDDOperationScope).
* *
* A stage may include one or more operation scopes if the RDD operations are * A stage may include one or more operation scopes if the RDD operations are
* streamlined into one stage (e.g. rdd.map(...).filter(...).flatMap(...)). * streamlined into one stage (e.g. rdd.map(...).filter(...).flatMap(...)).
@ -52,14 +52,7 @@
*/ */
var VizConstants = { var VizConstants = {
rddColor: "#444444", svgMarginX: 20,
rddCachedColor: "#FF0000",
rddOperationColor: "#AADFFF",
stageColor: "#FFDDEE",
clusterLabelColor: "#888888",
edgeColor: "#444444",
edgeWidth: "1.5px",
svgMarginX: 0,
svgMarginY: 20, svgMarginY: 20,
stageSep: 50, stageSep: 50,
graphPrefix: "graph_", graphPrefix: "graph_",
@ -69,13 +62,21 @@ var VizConstants = {
stageClusterPrefix: "cluster_stage_" stageClusterPrefix: "cluster_stage_"
}; };
// Helper d3 accessors for the elements that contain our graph and its metadata var JobPageVizConstants = {
function graphContainer() { return d3.select("#dag-viz-graph"); } clusterLabelSize: 11,
function metadataContainer() { return d3.select("#dag-viz-metadata"); } stageClusterLabelSize: 14
}
var StagePageVizConstants = {
clusterLabelSize: 14,
stageClusterLabelSize: 18
}
/* /*
* Show or hide the RDD DAG visualization. * Show or hide the RDD DAG visualization.
*
* The graph is only rendered the first time this is called. * The graph is only rendered the first time this is called.
* This is the narrow interface called from the Scala UI code.
*/ */
function toggleDagViz(forJob) { function toggleDagViz(forJob) {
var arrowSelector = ".expand-dag-viz-arrow"; var arrowSelector = ".expand-dag-viz-arrow";
@ -113,70 +114,52 @@ function toggleDagViz(forJob) {
function renderDagViz(forJob) { function renderDagViz(forJob) {
// If there is not a dot file to render, fail fast and report error // If there is not a dot file to render, fail fast and report error
var jobOrStage = forJob ? "job" : "stage";
if (metadataContainer().empty()) { if (metadataContainer().empty()) {
graphContainer().append("div").text( graphContainer()
"No visualization information available for this " + (forJob ? "job" : "stage")); .append("div")
.text("No visualization information available for this " + jobOrStage);
return; return;
} }
var svg = graphContainer().append("svg"); // Render
var svg = graphContainer()
.append("svg")
.attr("class", jobOrStage);
if (forJob) { if (forJob) {
renderDagVizForJob(svg); renderDagVizForJob(svg);
} else { } else {
renderDagVizForStage(svg); renderDagVizForStage(svg);
} }
// Find cached RDDs // Find cached RDDs and mark them as such
metadataContainer().selectAll(".cached-rdd").each(function(v) { metadataContainer().selectAll(".cached-rdd").each(function(v) {
var nodeId = VizConstants.nodePrefix + d3.select(this).text(); var nodeId = VizConstants.nodePrefix + d3.select(this).text();
graphContainer().selectAll("#" + nodeId).classed("cached", true); svg.selectAll("#" + nodeId).classed("cached", true);
}); });
// Set the appropriate SVG dimensions to ensure that all elements are displayed // More post-processing
var boundingBox = svg.node().getBBox(); drawClusterLabels(svg, forJob);
svg.style("width", (boundingBox.width + VizConstants.svgMarginX) + "px"); resizeSvg(svg);
svg.style("height", (boundingBox.height + VizConstants.svgMarginY) + "px");
// Add labels to clusters because dagre-d3 doesn't do this for us
svg.selectAll("g.cluster rect").each(function() {
var rect = d3.select(this);
var cluster = d3.select(this.parentNode);
// Shift the boxes up a little to make room for the labels
rect.attr("y", toFloat(rect.attr("y")) - 10);
rect.attr("height", toFloat(rect.attr("height")) + 10);
var labelX = toFloat(rect.attr("x")) + toFloat(rect.attr("width")) - 5;
var labelY = toFloat(rect.attr("y")) + 15;
var labelText = cluster.attr("name").replace(VizConstants.clusterPrefix, "");
cluster.append("text")
.attr("x", labelX)
.attr("y", labelY)
.attr("text-anchor", "end")
.text(labelText);
});
// We have shifted a few elements upwards, so we should fix the SVG views
var startX = -VizConstants.svgMarginX;
var startY = -VizConstants.svgMarginY;
var endX = toFloat(svg.style("width")) + VizConstants.svgMarginX;
var endY = toFloat(svg.style("height")) + VizConstants.svgMarginY;
var newViewBox = startX + " " + startY + " " + endX + " " + endY;
svg.attr("viewBox", newViewBox);
// Lastly, apply some custom style to the DAG
styleDagViz(forJob);
} }
/* Render the RDD DAG visualization for a stage. */ /* Render the RDD DAG visualization on the stage page. */
function renderDagVizForStage(svgContainer) { function renderDagVizForStage(svgContainer) {
var metadata = metadataContainer().select(".stage-metadata"); var metadata = metadataContainer().select(".stage-metadata");
var dot = metadata.select(".dot-file").text(); var dot = metadata.select(".dot-file").text();
var containerId = VizConstants.graphPrefix + metadata.attr("stageId"); var containerId = VizConstants.graphPrefix + metadata.attr("stage-id");
var container = svgContainer.append("g").attr("id", containerId); var container = svgContainer.append("g").attr("id", containerId);
renderDot(dot, container); renderDot(dot, container);
// Round corners on RDDs
svgContainer
.selectAll("g.node rect")
.attr("rx", "5")
.attr("ry", "5");
} }
/* /*
* Render the RDD DAG visualization for a job. * Render the RDD DAG visualization on the job page.
* *
* Due to limitations in dagre-d3, each stage is rendered independently so that * Due to limitations in dagre-d3, each stage is rendered independently so that
* we have more control on how to position them. Unfortunately, this means we * we have more control on how to position them. Unfortunately, this means we
@ -186,32 +169,46 @@ function renderDagVizForStage(svgContainer) {
function renderDagVizForJob(svgContainer) { function renderDagVizForJob(svgContainer) {
var crossStageEdges = []; var crossStageEdges = [];
// Each div.stage-metadata contains the information needed to generate the graph
// for a stage. This includes the DOT file produced from the appropriate UI listener,
// any incoming and outgoing edges, and any cached RDDs that belong to this stage.
metadataContainer().selectAll(".stage-metadata").each(function(d, i) { metadataContainer().selectAll(".stage-metadata").each(function(d, i) {
var metadata = d3.select(this); var metadata = d3.select(this);
var dot = metadata.select(".dot-file").text(); var dot = metadata.select(".dot-file").text();
var stageId = metadata.attr("stageId"); var stageId = metadata.attr("stage-id");
var containerId = VizConstants.graphPrefix + stageId; var containerId = VizConstants.graphPrefix + stageId;
// TODO: handle stage attempts // Link each graph to the corresponding stage page (TODO: handle stage attempts)
var stageLink = var stageLink =
"/stages/stage/?id=" + stageId.replace(VizConstants.stagePrefix, "") + "&attempt=0"; "/stages/stage/?id=" + stageId.replace(VizConstants.stagePrefix, "") + "&attempt=0";
var container = svgContainer var container = svgContainer
.append("a").attr("xlink:href", stageLink) .append("a")
.append("g").attr("id", containerId); .attr("xlink:href", stageLink)
// Now we need to shift the container for this stage so it doesn't overlap .append("g")
// with existing ones. We do not need to do this for the first stage. .attr("id", containerId);
// Now we need to shift the container for this stage so it doesn't overlap with
// existing ones, taking into account the position and width of the last stage's
// container. We do not need to do this for the first stage of this job.
if (i > 0) { if (i > 0) {
// Take into account the position and width of the last stage's container var existingStages = svgContainer
var existingStages = stageClusters(); .selectAll("g.cluster")
.filter("[id*=\"" + VizConstants.stageClusterPrefix + "\"]");
if (!existingStages.empty()) { if (!existingStages.empty()) {
var lastStage = existingStages[0].pop(); var lastStage = d3.select(existingStages[0].pop());
var lastStageId = d3.select(lastStage).attr("id"); var lastStageId = lastStage.attr("id");
var lastStageWidth = toFloat(d3.select("#" + lastStageId + " rect").attr("width")); var lastStageWidth = toFloat(svgContainer
var lastStagePosition = getAbsolutePosition(lastStageId); .select("#" + lastStageId)
.select("rect")
.attr("width"));
var lastStagePosition = getAbsolutePosition(lastStage);
var offset = lastStagePosition.x + lastStageWidth + VizConstants.stageSep; var offset = lastStagePosition.x + lastStageWidth + VizConstants.stageSep;
container.attr("transform", "translate(" + offset + ", 0)"); container.attr("transform", "translate(" + offset + ", 0)");
} }
} }
// Actually render the stage
renderDot(dot, container); renderDot(dot, container);
// If there are any incoming edges into this graph, keep track of them to render // If there are any incoming edges into this graph, keep track of them to render
// them separately later. Note that we cannot draw them now because we need to // them separately later. Note that we cannot draw them now because we need to
// put these edges in a separate container that is on top of all stage graphs. // put these edges in a separate container that is on top of all stage graphs.
@ -221,15 +218,7 @@ function renderDagVizForJob(svgContainer) {
}); });
}); });
// Draw edges that cross stages drawCrossStageEdges(crossStageEdges, svgContainer);
if (crossStageEdges.length > 0) {
var container = svgContainer.append("g").attr("id", "cross-stage-edges");
for (var i = 0; i < crossStageEdges.length; i++) {
var fromRDDId = crossStageEdges[i][0];
var toRDDId = crossStageEdges[i][1];
connectRDDs(fromRDDId, toRDDId, container);
}
}
} }
/* Render the dot file as an SVG in the given container. */ /* Render the dot file as an SVG in the given container. */
@ -243,99 +232,156 @@ function renderDot(dot, container) {
renderer(container, g); renderer(container, g);
} }
/* Style the visualization we just rendered. */ /* -------------------- *
function styleDagViz(forJob) { * | Helper functions | *
graphContainer().selectAll("svg g.cluster rect") * -------------------- */
.style("fill", "white")
.style("stroke", VizConstants.rddOperationColor)
.style("stroke-width", "4px")
.style("stroke-opacity", "0.5");
graphContainer().selectAll("svg g.cluster text")
.attr("fill", VizConstants.clusterLabelColor)
.attr("font-size", "11px");
graphContainer().selectAll("svg path")
.style("stroke", VizConstants.edgeColor)
.style("stroke-width", VizConstants.edgeWidth);
stageClusters()
.select("rect")
.style("stroke", VizConstants.stageColor)
.style("strokeWidth", "6px");
// Put an arrow at the end of every edge // Helper d3 accessors
// We need to do this because we manually render some edges ourselves function graphContainer() { return d3.select("#dag-viz-graph"); }
// For these edges, we borrow the arrow marker generated by dagre-d3 function metadataContainer() { return d3.select("#dag-viz-metadata"); }
var dagreD3Marker = graphContainer().select("svg g.edgePaths marker").node();
graphContainer().select("svg")
.append(function() { return dagreD3Marker.cloneNode(true); })
.attr("id", "marker-arrow")
.select("path")
.attr("fill", VizConstants.edgeColor)
.attr("strokeWidth", "0px");
graphContainer().selectAll("svg g > path").attr("marker-end", "url(#marker-arrow)");
graphContainer().selectAll("svg g.edgePaths def").remove(); // We no longer need these
// Apply any job or stage specific styles /*
* Helper function to create draw a label for each cluster.
*
* We need to do this manually because dagre-d3 does not support labeling clusters.
* In general, the clustering support for dagre-d3 is quite limited at this point.
*/
function drawClusterLabels(svgContainer, forJob) {
if (forJob) { if (forJob) {
styleDagVizForJob(); var clusterLabelSize = JobPageVizConstants.clusterLabelSize;
var stageClusterLabelSize = JobPageVizConstants.stageClusterLabelSize;
} else { } else {
styleDagVizForStage(); var clusterLabelSize = StagePageVizConstants.clusterLabelSize;
var stageClusterLabelSize = StagePageVizConstants.stageClusterLabelSize;
} }
} svgContainer.selectAll("g.cluster").each(function() {
var cluster = d3.select(this);
/* Apply job-page-specific style to the visualization. */ var isStage = cluster.attr("id").indexOf(VizConstants.stageClusterPrefix) > -1;
function styleDagVizForJob() { var labelSize = isStage ? stageClusterLabelSize : clusterLabelSize;
graphContainer().selectAll("svg g.node circle") drawClusterLabel(cluster, labelSize);
.style("fill", VizConstants.rddColor); });
// TODO: add a legend to explain what a highlighted dot means
graphContainer().selectAll("svg g.cached circle")
.style("fill", VizConstants.rddCachedColor);
graphContainer().selectAll("svg g#cross-stage-edges path")
.style("fill", "none");
}
/* Apply stage-page-specific style to the visualization. */
function styleDagVizForStage() {
graphContainer().selectAll("svg g.node rect")
.style("fill", "none")
.style("stroke", VizConstants.rddColor)
.style("stroke-width", "2px")
.attr("rx", "5") // round corners
.attr("ry", "5");
// TODO: add a legend to explain what a highlighted RDD means
graphContainer().selectAll("svg g.cached rect")
.style("stroke", VizConstants.rddCachedColor);
graphContainer().selectAll("svg g.node g.label text tspan")
.style("fill", VizConstants.rddColor);
} }
/* /*
* (Job page only) Helper method to compute the absolute * Helper function to draw a label for the given cluster element based on its name.
* position of the group element identified by the given ID. *
* In the process, we need to expand the bounding box to make room for the label.
* We need to do this because dagre-d3 did not take this into account when it first
* rendered the bounding boxes. Note that this means we need to adjust the view box
* of the SVG afterwards since we shifted a few boxes around.
*/ */
function getAbsolutePosition(groupId) { function drawClusterLabel(d3cluster, fontSize) {
var obj = d3.select("#" + groupId).filter("g"); var cluster = d3cluster;
var _x = 0, _y = 0; var rect = d3cluster.select("rect");
rect.attr("y", toFloat(rect.attr("y")) - fontSize);
rect.attr("height", toFloat(rect.attr("height")) + fontSize);
var labelX = toFloat(rect.attr("x")) + toFloat(rect.attr("width")) - fontSize / 2;
var labelY = toFloat(rect.attr("y")) + fontSize * 1.5;
var labelText = cluster.attr("name").replace(VizConstants.clusterPrefix, "");
cluster.append("text")
.attr("x", labelX)
.attr("y", labelY)
.attr("text-anchor", "end")
.style("font-size", fontSize)
.text(labelText);
}
/*
* Helper function to size the SVG appropriately such that all elements are displyed.
* This assumes that all outermost elements are clusters (rectangles).
*/
function resizeSvg(svg) {
var allClusters = svg.selectAll("g.cluster rect")[0];
var startX = -VizConstants.svgMarginX +
toFloat(d3.min(allClusters, function(e) {
return getAbsolutePosition(d3.select(e)).x;
}));
var startY = -VizConstants.svgMarginY +
toFloat(d3.min(allClusters, function(e) {
return getAbsolutePosition(d3.select(e)).y;
}));
var endX = VizConstants.svgMarginX +
toFloat(d3.max(allClusters, function(e) {
var t = d3.select(e)
return getAbsolutePosition(t).x + toFloat(t.attr("width"));
}));
var endY = VizConstants.svgMarginY +
toFloat(d3.max(allClusters, function(e) {
var t = d3.select(e)
return getAbsolutePosition(t).y + toFloat(t.attr("height"));
}));
var width = endX - startX;
var height = endY - startY;
svg.attr("viewBox", startX + " " + startY + " " + width + " " + height)
.attr("width", width)
.attr("height", height);
}
/*
* (Job page only) Helper function to draw edges that cross stage boundaries.
* We need to do this manually because we render each stage separately in dagre-d3.
*/
function drawCrossStageEdges(edges, svgContainer) {
if (edges.length == 0) {
return;
}
// Draw the paths first
var edgesContainer = svgContainer.append("g").attr("id", "cross-stage-edges");
for (var i = 0; i < edges.length; i++) {
var fromRDDId = edges[i][0];
var toRDDId = edges[i][1];
connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer);
}
// Now draw the arrows by borrowing the arrow marker generated by dagre-d3
var dagreD3Marker = svgContainer.select("g.edgePaths marker").node();
if (!dagreD3Marker.empty()) {
svgContainer
.append(function() { return dagreD3Marker.cloneNode(true); })
.attr("id", "marker-arrow")
svgContainer.selectAll("g > path").attr("marker-end", "url(#marker-arrow)");
svgContainer.selectAll("g.edgePaths def").remove(); // We no longer need these
}
}
/*
* (Job page only) Helper function to compute the absolute
* position of the specified element in our graph.
*/
function getAbsolutePosition(d3selection) {
if (d3selection.empty()) {
throw "Attempted to get absolute position of an empty selection.";
}
var obj = d3selection;
var _x = toFloat(obj.attr("x")) || 0;
var _y = toFloat(obj.attr("y")) || 0;
while (!obj.empty()) { while (!obj.empty()) {
var transformText = obj.attr("transform"); var transformText = obj.attr("transform");
var translate = d3.transform(transformText).translate if (transformText) {
_x += translate[0]; var translate = d3.transform(transformText).translate;
_y += translate[1]; _x += toFloat(translate[0]);
obj = d3.select(obj.node().parentNode).filter("g") _y += toFloat(translate[1]);
}
// Climb upwards to find how our parents are translated
obj = d3.select(obj.node().parentNode);
// Stop when we've reached the graph container itself
if (obj.node() == graphContainer().node()) {
break;
}
} }
return { x: _x, y: _y }; return { x: _x, y: _y };
} }
/* (Job page only) Connect two RDD nodes with a curved edge. */ /* (Job page only) Helper function to connect two RDDs with a curved edge. */
function connectRDDs(fromRDDId, toRDDId, container) { function connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer) {
var fromNodeId = VizConstants.nodePrefix + fromRDDId; var fromNodeId = VizConstants.nodePrefix + fromRDDId;
var toNodeId = VizConstants.nodePrefix + toRDDId var toNodeId = VizConstants.nodePrefix + toRDDId;
var fromPos = getAbsolutePosition(fromNodeId); var fromPos = getAbsolutePosition(svgContainer.select("#" + fromNodeId));
var toPos = getAbsolutePosition(toNodeId); var toPos = getAbsolutePosition(svgContainer.select("#" + toNodeId));
// On the job page, RDDs are rendered as dots (circles). When rendering the path, // On the job page, RDDs are rendered as dots (circles). When rendering the path,
// we need to account for the radii of these circles. Otherwise the arrow heads // we need to account for the radii of these circles. Otherwise the arrow heads
// will bleed into the circle itself. // will bleed into the circle itself.
var delta = toFloat(graphContainer() var delta = toFloat(svgContainer
.select("g.node#" + toNodeId) .select("g.node#" + toNodeId)
.select("circle") .select("circle")
.attr("r")); .attr("r"));
@ -375,18 +421,15 @@ function connectRDDs(fromRDDId, toRDDId, container) {
} }
var line = d3.svg.line().interpolate("basis"); var line = d3.svg.line().interpolate("basis");
container.append("path").datum(points).attr("d", line); edgesContainer.append("path").datum(points).attr("d", line);
} }
/* Helper d3 accessor to clusters that represent stages. */ /* Helper function to convert attributes to numeric values. */
function stageClusters() {
return graphContainer().selectAll("g.cluster").filter(function() {
return d3.select(this).attr("id").indexOf(VizConstants.stageClusterPrefix) > -1;
});
}
/* Helper method to convert attributes to numeric values. */
function toFloat(f) { function toFloat(f) {
return parseFloat(f.replace(/px$/, "")); if (f) {
return parseFloat(f.toString().replace(/px$/, ""));
} else {
return f;
}
} }

14
core/src/main/scala/org/apache/spark/ui/UIUtils.scala
View file

@ -156,13 +156,10 @@ private[spark] object UIUtils extends Logging {
def commonHeaderNodes: Seq[Node] = { def commonHeaderNodes: Seq[Node] = {
<meta http-equiv="Content-type" content="text/html; charset=utf-8" /> <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")} <link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")} type="text/css" />
type="text/css" /> <link rel="stylesheet" href={prependBaseUri("/static/webui.css")} type="text/css" />
<link rel="stylesheet" href={prependBaseUri("/static/webui.css")} <link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")} type="text/css" />
type="text/css" /> <link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")} type="text/css" />
<link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")}
typ="text/css" />
<link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")}></link>
<script src={prependBaseUri("/static/sorttable.js")} ></script> <script src={prependBaseUri("/static/sorttable.js")} ></script>
<script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script> <script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script>
<script src={prependBaseUri("/static/vis.min.js")}></script> <script src={prependBaseUri("/static/vis.min.js")}></script>
@ -174,6 +171,7 @@ private[spark] object UIUtils extends Logging {
} }
def vizHeaderNodes: Seq[Node] = { def vizHeaderNodes: Seq[Node] = {
<link rel="stylesheet" href={prependBaseUri("/static/spark-dag-viz.css")} type="text/css" />
<script src={prependBaseUri("/static/d3.min.js")}></script> <script src={prependBaseUri("/static/d3.min.js")}></script>
<script src={prependBaseUri("/static/dagre-d3.min.js")}></script> <script src={prependBaseUri("/static/dagre-d3.min.js")}></script>
<script src={prependBaseUri("/static/graphlib-dot.min.js")}></script> <script src={prependBaseUri("/static/graphlib-dot.min.js")}></script>
@ -358,7 +356,7 @@ private[spark] object UIUtils extends Logging {
<div id="dag-viz-metadata"> <div id="dag-viz-metadata">
{ {
graphs.map { g => graphs.map { g =>
<div class="stage-metadata" stageId={g.rootCluster.id} style="display:none"> <div class="stage-metadata" stage-id={g.rootCluster.id} style="display:none">
<div class="dot-file">{RDDOperationGraph.makeDotFile(g, forJob)}</div> <div class="dot-file">{RDDOperationGraph.makeDotFile(g, forJob)}</div>
{ g.incomingEdges.map { e => <div class="incoming-edge">{e.fromId},{e.toId}</div> } } { g.incomingEdges.map { e => <div class="incoming-edge">{e.fromId},{e.toId}</div> } }
{ g.outgoingEdges.map { e => <div class="outgoing-edge">{e.fromId},{e.toId}</div> } } { g.outgoingEdges.map { e => <div class="outgoing-edge">{e.fromId},{e.toId}</div> } }

19
core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
View file

@ -181,22 +181,29 @@ private[ui] object RDDOperationGraph extends Logging {
if (forJob) { if (forJob) {
s"""${node.id} [label=" " shape="circle" padding="5" labelStyle="font-size: 0"]""" s"""${node.id} [label=" " shape="circle" padding="5" labelStyle="font-size: 0"]"""
} else { } else {
s"""${node.id} [label="${node.name} (${node.id})"]""" s"""${node.id} [label="${node.name} (${node.id})" padding="5" labelStyle="font-size: 10"]"""
} }
} }
/** Return the dot representation of a subgraph in an RDDOperationGraph. */ /** Return the dot representation of a subgraph in an RDDOperationGraph. */
private def makeDotSubgraph( private def makeDotSubgraph(
scope: RDDOperationCluster, cluster: RDDOperationCluster,
forJob: Boolean, forJob: Boolean,
indent: String): String = { indent: String): String = {
val subgraph = new StringBuilder val subgraph = new StringBuilder
subgraph.append(indent + s"subgraph cluster${scope.id} {\n") // TODO: move specific graph properties like these to spark-dag-viz.js
subgraph.append(indent + s""" label="${scope.name}";\n""") val paddingTop = if (forJob) 10 else 20
scope.childNodes.foreach { node => subgraph.append(indent + s"subgraph cluster${cluster.id} {\n")
subgraph.append(indent + s""" label="${cluster.name}";\n""")
// If there are nested clusters, add some padding
// Do this for the stage page because we use bigger fonts there
if (cluster.childClusters.nonEmpty) {
subgraph.append(indent + s""" paddingTop="$paddingTop";\n""")
}
cluster.childNodes.foreach { node =>
subgraph.append(indent + s" ${makeDotNode(node, forJob)};\n") subgraph.append(indent + s" ${makeDotNode(node, forJob)};\n")
} }
scope.childClusters.foreach { cscope => cluster.childClusters.foreach { cscope =>
subgraph.append(makeDotSubgraph(cscope, forJob, indent + " ")) subgraph.append(makeDotSubgraph(cscope, forJob, indent + " "))
} }
subgraph.append(indent + "}\n") subgraph.append(indent + "}\n")