[SPARK-23475][UI] Show also skipped stages
## What changes were proposed in this pull request? SPARK-20648 introduced the status `SKIPPED` for the stages. On the UI, previously, skipped stages were shown as `PENDING`; after this change, they are not shown on the UI. The PR introduce a new section in order to show also `SKIPPED` stages in a proper table. ## How was this patch tested? manual tests Author: Marco Gaido <marcogaido91@gmail.com> Closes #20651 from mgaido91/SPARK-23475.
This commit is contained in:
parent
45cf714ee6
commit
87293c746e
|
@ -72,6 +72,7 @@ $(function() {
|
|||
collapseTablePageLoad('collapse-aggregated-allActiveStages','aggregated-allActiveStages');
|
||||
collapseTablePageLoad('collapse-aggregated-allPendingStages','aggregated-allPendingStages');
|
||||
collapseTablePageLoad('collapse-aggregated-allCompletedStages','aggregated-allCompletedStages');
|
||||
collapseTablePageLoad('collapse-aggregated-allSkippedStages','aggregated-allSkippedStages');
|
||||
collapseTablePageLoad('collapse-aggregated-allFailedStages','aggregated-allFailedStages');
|
||||
collapseTablePageLoad('collapse-aggregated-activeStages','aggregated-activeStages');
|
||||
collapseTablePageLoad('collapse-aggregated-pendingOrSkippedStages','aggregated-pendingOrSkippedStages');
|
||||
|
|
|
@ -36,6 +36,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
|
|||
|
||||
val activeStages = allStages.filter(_.status == StageStatus.ACTIVE)
|
||||
val pendingStages = allStages.filter(_.status == StageStatus.PENDING)
|
||||
val skippedStages = allStages.filter(_.status == StageStatus.SKIPPED)
|
||||
val completedStages = allStages.filter(_.status == StageStatus.COMPLETE)
|
||||
val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse
|
||||
|
||||
|
@ -51,6 +52,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
|
|||
val completedStagesTable =
|
||||
new StageTableBase(parent.store, request, completedStages, "completed", "completedStage",
|
||||
parent.basePath, subPath, parent.isFairScheduler, false, false)
|
||||
val skippedStagesTable =
|
||||
new StageTableBase(parent.store, request, skippedStages, "skipped", "skippedStage",
|
||||
parent.basePath, subPath, parent.isFairScheduler, false, false)
|
||||
val failedStagesTable =
|
||||
new StageTableBase(parent.store, request, failedStages, "failed", "failedStage",
|
||||
parent.basePath, subPath, parent.isFairScheduler, false, true)
|
||||
|
@ -66,6 +70,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
|
|||
val shouldShowActiveStages = activeStages.nonEmpty
|
||||
val shouldShowPendingStages = pendingStages.nonEmpty
|
||||
val shouldShowCompletedStages = completedStages.nonEmpty
|
||||
val shouldShowSkippedStages = skippedStages.nonEmpty
|
||||
val shouldShowFailedStages = failedStages.nonEmpty
|
||||
|
||||
val appSummary = parent.store.appSummary()
|
||||
|
@ -102,6 +107,14 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
|
|||
</li>
|
||||
}
|
||||
}
|
||||
{
|
||||
if (shouldShowSkippedStages) {
|
||||
<li id="completed-summary">
|
||||
<a href="#skipped"><strong>Skipped Stages:</strong></a>
|
||||
{skippedStages.size}
|
||||
</li>
|
||||
}
|
||||
}
|
||||
{
|
||||
if (shouldShowFailedStages) {
|
||||
<li>
|
||||
|
@ -172,6 +185,20 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
|
|||
{completedStagesTable.toNodeSeq}
|
||||
</div>
|
||||
}
|
||||
if (shouldShowSkippedStages) {
|
||||
content ++=
|
||||
<span id="skipped" class="collapse-aggregated-allSkippedStages collapse-table"
|
||||
onClick="collapseTable('collapse-aggregated-allSkippedStages',
|
||||
'aggregated-allSkippedStages')">
|
||||
<h4>
|
||||
<span class="collapse-table-arrow arrow-open"></span>
|
||||
<a>Skipped Stages ({skippedStages.size})</a>
|
||||
</h4>
|
||||
</span> ++
|
||||
<div class="aggregated-allSkippedStages collapsible-table">
|
||||
{skippedStagesTable.toNodeSeq}
|
||||
</div>
|
||||
}
|
||||
if (shouldShowFailedStages) {
|
||||
content ++=
|
||||
<span id ="failed" class="collapse-aggregated-allFailedStages collapse-table"
|
||||
|
|
|
@ -707,6 +707,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
|
|||
}
|
||||
}
|
||||
|
||||
test("stages page should show skipped stages") {
|
||||
withSpark(newSparkContext()) { sc =>
|
||||
val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache()
|
||||
rdd.count()
|
||||
rdd.count()
|
||||
|
||||
eventually(timeout(5 seconds), interval(50 milliseconds)) {
|
||||
goToUi(sc, "/stages")
|
||||
find(id("skipped")).get.text should be("Skipped Stages (1)")
|
||||
}
|
||||
val stagesJson = getJson(sc.ui.get, "stages")
|
||||
stagesJson.children.size should be (4)
|
||||
val stagesStatus = stagesJson.children.map(_ \ "status")
|
||||
stagesStatus.count(_ == JString(StageStatus.SKIPPED.name())) should be (1)
|
||||
}
|
||||
}
|
||||
|
||||
def goToUi(sc: SparkContext, path: String): Unit = {
|
||||
goToUi(sc.ui.get, path)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue