ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #826 from kayousterhout/ui_fix

Browse source 
Fixed 2 bugs in executor UI (incl. SPARK-877)
This commit is contained in:
Reynold Xin 2013-08-14 00:17:07 -07:00
parent 3f14cbab05 a88aa5e6ed
commit 63446f9208
1 changed files with 10 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
core/src/main/scala/spark/ui/exec/ExecutorsUI.scala
View file

@ -38,10 +38,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
def render(request: HttpServletRequest): Seq[Node] = { def render(request: HttpServletRequest): Seq[Node] = {
val storageStatusList = sc.getExecutorStorageStatus val storageStatusList = sc.getExecutorStorageStatus
val maxMem = storageStatusList.map(_.maxMem).reduce(_+_) val maxMem = storageStatusList.map(_.maxMem).fold(0L)(_+_)
val memUsed = storageStatusList.map(_.memUsed()).reduce(_+_) val memUsed = storageStatusList.map(_.memUsed()).fold(0L)(_+_)
val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize)) val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize)).fold(0L)(_+_)
.reduceOption(_+_).getOrElse(0L)
val execHead = Seq("Executor ID", "Address", "RDD blocks", "Memory used", "Disk used", val execHead = Seq("Executor ID", "Address", "RDD blocks", "Memory used", "Disk used",
"Active tasks", "Failed tasks", "Complete tasks", "Total tasks") "Active tasks", "Failed tasks", "Complete tasks", "Total tasks")
@ -93,10 +92,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString
val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString
val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString
val activeTasks = listener.executorToTasksActive.get(a.toString).map(l => l.size) val activeTasks = listener.executorToTasksActive.get(a.toString).map(l => l.size).getOrElse(0)
.getOrElse(0).toString val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0)
val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0).toString val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0)
val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0).toString
val totalTasks = activeTasks + failedTasks + completedTasks val totalTasks = activeTasks + failedTasks + completedTasks
Seq( Seq(
@ -106,10 +104,10 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
memUsed, memUsed,
maxMem, maxMem,
diskUsed, diskUsed,
activeTasks, activeTasks.toString,
failedTasks, failedTasks.toString,
completedTasks, completedTasks.toString,
totalTasks totalTasks.toString
) )
} }