[SPARK-10930] History "Stages" page "duration" can be confusing
Author: Derek Dagit <derekd@yahoo-inc.com> Closes #9051 from d2r/spark-10930-ui-max-task-dur.
This commit is contained in:
parent
3a9851936d
commit
c07a50b862
|
@ -145,9 +145,22 @@ private[ui] class StageTableBase(
|
||||||
case None => "Unknown"
|
case None => "Unknown"
|
||||||
}
|
}
|
||||||
val finishTime = s.completionTime.getOrElse(System.currentTimeMillis)
|
val finishTime = s.completionTime.getOrElse(System.currentTimeMillis)
|
||||||
val duration = s.submissionTime.map { t =>
|
|
||||||
if (finishTime > t) finishTime - t else System.currentTimeMillis - t
|
// The submission time for a stage is misleading because it counts the time
|
||||||
}
|
// the stage waits to be launched. (SPARK-10930)
|
||||||
|
val taskLaunchTimes =
|
||||||
|
stageData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0)
|
||||||
|
val duration: Option[Long] =
|
||||||
|
if (taskLaunchTimes.nonEmpty) {
|
||||||
|
val startTime = taskLaunchTimes.min
|
||||||
|
if (finishTime > startTime) {
|
||||||
|
Some(finishTime - startTime)
|
||||||
|
} else {
|
||||||
|
Some(System.currentTimeMillis() - startTime)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
|
val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
|
||||||
|
|
||||||
val inputRead = stageData.inputBytes
|
val inputRead = stageData.inputBytes
|
||||||
|
|
Loading…
Reference in a new issue