ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-24851][UI] Map a Stage ID to it's Associated Job ID

Browse source 
It would be nice to have a field in Stage Page UI which would show mapping of the current stage id to the job id's to which that stage belongs to.

## What changes were proposed in this pull request?

Added a field in Stage UI to display the corresponding job id for that particular stage.

## How was this patch tested?

<img width="448" alt="screen shot 2018-07-25 at 1 33 07 pm" src="https://user-images.githubusercontent.com/22228190/43220447-a8e94f80-900f-11e8-8a20-a235bbd5a369.png">

Closes #21809 from pgandhi999/SPARK-24851.

Authored-by: pgandhi <pgandhi@oath.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
This commit is contained in:
pgandhi 2018-10-09 08:59:21 -05:00 committed by Thomas Graves
parent e3133f4abf
commit deb9588b2a
3 changed files with 17 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
View file

@ -112,10 +112,12 @@ private[spark] class AppStatusStore(
}
}
def stageAttempt(stageId: Int, stageAttemptId: Int, details: Boolean = false): v1.StageData = {
def stageAttempt(stageId: Int, stageAttemptId: Int,
details: Boolean = false): (v1.StageData, Seq[Int]) = {
val stageKey = Array(stageId, stageAttemptId)
val stage = store.read(classOf[StageDataWrapper], stageKey).info
if (details) stageWithDetails(stage) else stage
val stageDataWrapper = store.read(classOf[StageDataWrapper], stageKey)
val stage = if (details) stageWithDetails(stageDataWrapper.info) else stageDataWrapper.info
(stage, stageDataWrapper.jobIds.toSeq)
}
def taskCount(stageId: Int, stageAttemptId: Int): Long = {

2
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala
View file

@ -56,7 +56,7 @@ private[v1] class StagesResource extends BaseAppResource {
@PathParam("stageAttemptId") stageAttemptId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean): StageData = withUI { ui =>
try {
ui.store.stageAttempt(stageId, stageAttemptId, details = details)
ui.store.stageAttempt(stageId, stageAttemptId, details = details)._1
} catch {
case _: NoSuchElementException =>
// Change the message depending on whether there are any attempts for the requested stage.

13
core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
View file

@ -105,7 +105,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
val stageAttemptId = parameterAttempt.toInt
val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)"
val stageData = parent.store
val (stageData, stageJobIds) = parent.store
.asOption(parent.store.stageAttempt(stageId, stageAttemptId, details = false))
.getOrElse {
val content =
@ -183,6 +183,15 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
{Utils.bytesToString(stageData.diskBytesSpilled)}
</li>
}}
{if (!stageJobIds.isEmpty) {
<li>
<strong>Associated Job Ids: </strong>
{stageJobIds.map(jobId => {val detailUrl = "%s/jobs/job/?id=%s".format(
UIUtils.prependBaseUri(request, parent.basePath), jobId)
<a href={s"${detailUrl}"}>{s"${jobId}"} &nbsp;&nbsp;</a>
})}
</li>
}}
</ul>
</div>
@ -1048,7 +1057,7 @@ private[ui] object ApiHelper {
}
def lastStageNameAndDescription(store: AppStatusStore, job: JobData): (String, String) = {
val stage = store.asOption(store.stageAttempt(job.stageIds.max, 0))
val stage = store.asOption(store.stageAttempt(job.stageIds.max, 0)._1)
(stage.map(_.name).getOrElse(""), stage.flatMap(_.description).getOrElse(job.name))
}