[SPARK-10930] History "Stages" page "duration" can be confusing
Author: Derek Dagit <derekd@yahoo-inc.com> Closes #9051 from d2r/spark-10930-ui-max-task-dur.
This commit is contained in:
parent
3a9851936d
commit
c07a50b862
|
@ -145,9 +145,22 @@ private[ui] class StageTableBase(
|
|||
case None => "Unknown"
|
||||
}
|
||||
val finishTime = s.completionTime.getOrElse(System.currentTimeMillis)
|
||||
val duration = s.submissionTime.map { t =>
|
||||
if (finishTime > t) finishTime - t else System.currentTimeMillis - t
|
||||
}
|
||||
|
||||
// The submission time for a stage is misleading because it counts the time
|
||||
// the stage waits to be launched. (SPARK-10930)
|
||||
val taskLaunchTimes =
|
||||
stageData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0)
|
||||
val duration: Option[Long] =
|
||||
if (taskLaunchTimes.nonEmpty) {
|
||||
val startTime = taskLaunchTimes.min
|
||||
if (finishTime > startTime) {
|
||||
Some(finishTime - startTime)
|
||||
} else {
|
||||
Some(System.currentTimeMillis() - startTime)
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
|
||||
|
||||
val inputRead = stageData.inputBytes
|
||||
|
|
Loading…
Reference in a new issue