Merge pull request #719 from karenfeng/ui-808
Creates Executors tab for Jobs UI
This commit is contained in:
commit
401aac8b18
|
@ -17,4 +17,4 @@
|
||||||
|
|
||||||
package spark.ui
|
package spark.ui
|
||||||
|
|
||||||
private[spark] object Page extends Enumeration { val Storage, Jobs, Environment = Value }
|
private[spark] object Page extends Enumeration { val Storage, Jobs, Environment, Executors = Value }
|
||||||
|
|
|
@ -23,6 +23,7 @@ import org.eclipse.jetty.server.{Handler, Server}
|
||||||
|
|
||||||
import spark.{Logging, SparkContext, Utils}
|
import spark.{Logging, SparkContext, Utils}
|
||||||
import spark.ui.env.EnvironmentUI
|
import spark.ui.env.EnvironmentUI
|
||||||
|
import spark.ui.exec.ExecutorsUI
|
||||||
import spark.ui.storage.BlockManagerUI
|
import spark.ui.storage.BlockManagerUI
|
||||||
import spark.ui.jobs.JobProgressUI
|
import spark.ui.jobs.JobProgressUI
|
||||||
import spark.ui.JettyUtils._
|
import spark.ui.JettyUtils._
|
||||||
|
@ -41,7 +42,9 @@ private[spark] class SparkUI(sc: SparkContext) extends Logging {
|
||||||
val storage = new BlockManagerUI(sc)
|
val storage = new BlockManagerUI(sc)
|
||||||
val jobs = new JobProgressUI(sc)
|
val jobs = new JobProgressUI(sc)
|
||||||
val env = new EnvironmentUI(sc)
|
val env = new EnvironmentUI(sc)
|
||||||
val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ env.getHandlers ++ handlers
|
val exec = new ExecutorsUI(sc)
|
||||||
|
val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ env.getHandlers ++
|
||||||
|
exec.getHandlers ++ handlers
|
||||||
|
|
||||||
/** Bind the HTTP server which backs this web interface */
|
/** Bind the HTTP server which backs this web interface */
|
||||||
def bind() {
|
def bind() {
|
||||||
|
@ -64,6 +67,7 @@ private[spark] class SparkUI(sc: SparkContext) extends Logging {
|
||||||
// This server must register all handlers, including JobProgressUI, before binding
|
// This server must register all handlers, including JobProgressUI, before binding
|
||||||
// JobProgressUI registers a listener with SparkContext, which requires sc to initialize
|
// JobProgressUI registers a listener with SparkContext, which requires sc to initialize
|
||||||
jobs.start()
|
jobs.start()
|
||||||
|
exec.start()
|
||||||
}
|
}
|
||||||
|
|
||||||
def stop() {
|
def stop() {
|
||||||
|
|
|
@ -40,6 +40,10 @@ private[spark] object UIUtils {
|
||||||
case Environment => <li class="active"><a href="/environment">Environment</a></li>
|
case Environment => <li class="active"><a href="/environment">Environment</a></li>
|
||||||
case _ => <li><a href="/environment">Environment</a></li>
|
case _ => <li><a href="/environment">Environment</a></li>
|
||||||
}
|
}
|
||||||
|
val executors = page match {
|
||||||
|
case Executors => <li class="active"><a href="/executors">Executors</a></li>
|
||||||
|
case _ => <li><a href="/executors">Executors</a></li>
|
||||||
|
}
|
||||||
|
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
|
@ -66,6 +70,7 @@ private[spark] object UIUtils {
|
||||||
{storage}
|
{storage}
|
||||||
{jobs}
|
{jobs}
|
||||||
{environment}
|
{environment}
|
||||||
|
{executors}
|
||||||
</ul>
|
</ul>
|
||||||
<ul id="infolist">
|
<ul id="infolist">
|
||||||
<li>Application: <strong>{sc.appName}</strong></li>
|
<li>Application: <strong>{sc.appName}</strong></li>
|
||||||
|
|
136
core/src/main/scala/spark/ui/exec/ExecutorsUI.scala
Normal file
136
core/src/main/scala/spark/ui/exec/ExecutorsUI.scala
Normal file
|
@ -0,0 +1,136 @@
|
||||||
|
package spark.ui.exec
|
||||||
|
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletRequest
|
||||||
|
|
||||||
|
import org.eclipse.jetty.server.Handler
|
||||||
|
|
||||||
|
import scala.collection.mutable.{ArrayBuffer, HashMap}
|
||||||
|
import scala.util.Properties
|
||||||
|
|
||||||
|
import spark.{ExceptionFailure, Logging, SparkContext, Success, Utils}
|
||||||
|
import spark.executor.TaskMetrics
|
||||||
|
import spark.scheduler.cluster.TaskInfo
|
||||||
|
import spark.scheduler._
|
||||||
|
import spark.SparkContext
|
||||||
|
import spark.storage.{StorageStatus, StorageUtils}
|
||||||
|
import spark.ui.JettyUtils._
|
||||||
|
import spark.ui.Page.Executors
|
||||||
|
import spark.ui.UIUtils.headerSparkPage
|
||||||
|
import spark.ui.UIUtils
|
||||||
|
import spark.Utils
|
||||||
|
|
||||||
|
import scala.xml.{Node, XML}
|
||||||
|
|
||||||
|
private[spark] class ExecutorsUI(val sc: SparkContext) {
|
||||||
|
|
||||||
|
private var _listener: Option[ExecutorsListener] = None
|
||||||
|
def listener = _listener.get
|
||||||
|
|
||||||
|
def start() {
|
||||||
|
_listener = Some(new ExecutorsListener)
|
||||||
|
sc.addSparkListener(listener)
|
||||||
|
}
|
||||||
|
|
||||||
|
def getHandlers = Seq[(String, Handler)](
|
||||||
|
("/executors", (request: HttpServletRequest) => render(request))
|
||||||
|
)
|
||||||
|
|
||||||
|
def render(request: HttpServletRequest): Seq[Node] = {
|
||||||
|
val storageStatusList = sc.getExecutorStorageStatus
|
||||||
|
|
||||||
|
val maxMem = storageStatusList.map(_.maxMem).reduce(_+_)
|
||||||
|
val memUsed = storageStatusList.map(_.memUsed()).reduce(_+_)
|
||||||
|
val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize))
|
||||||
|
.reduceOption(_+_).getOrElse(0L)
|
||||||
|
|
||||||
|
val execHead = Seq("Executor ID", "Address", "RDD blocks", "Memory used", "Disk used",
|
||||||
|
"Failed tasks", "Complete tasks", "Total tasks")
|
||||||
|
def execRow(kv: Seq[String]) =
|
||||||
|
<tr>
|
||||||
|
<td>{kv(0)}</td>
|
||||||
|
<td>{kv(1)}</td>
|
||||||
|
<td>{kv(2)}</td>
|
||||||
|
<td sorttable_customkey={kv(3)}>
|
||||||
|
{Utils.memoryBytesToString(kv(3).toLong)} / {Utils.memoryBytesToString(kv(4).toLong)}
|
||||||
|
</td>
|
||||||
|
<td sorttable_customkey={kv(5)}>
|
||||||
|
{Utils.memoryBytesToString(kv(5).toLong)}
|
||||||
|
</td>
|
||||||
|
<td>{kv(6)}</td>
|
||||||
|
<td>{kv(7)}</td>
|
||||||
|
<td>{kv(8)}</td>
|
||||||
|
</tr>
|
||||||
|
val execInfo =
|
||||||
|
for (b <- 0 until storageStatusList.size)
|
||||||
|
yield getExecInfo(b)
|
||||||
|
val execTable = UIUtils.listingTable(execHead, execRow, execInfo)
|
||||||
|
|
||||||
|
val content =
|
||||||
|
<div class="row">
|
||||||
|
<div class="span12">
|
||||||
|
<ul class="unstyled">
|
||||||
|
<li><strong>Memory:</strong>
|
||||||
|
{Utils.memoryBytesToString(memUsed)} Used
|
||||||
|
({Utils.memoryBytesToString(maxMem)} Total) </li>
|
||||||
|
<li><strong>Disk:</strong> {Utils.memoryBytesToString(diskSpaceUsed)} Used </li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class = "row">
|
||||||
|
<div class="span12">
|
||||||
|
{execTable}
|
||||||
|
</div>
|
||||||
|
</div>;
|
||||||
|
|
||||||
|
headerSparkPage(content, sc, "Executors", Executors)
|
||||||
|
}
|
||||||
|
|
||||||
|
def getExecInfo(a: Int): Seq[String] = {
|
||||||
|
val execId = sc.getExecutorStorageStatus(a).blockManagerId.executorId
|
||||||
|
val hostPort = sc.getExecutorStorageStatus(a).blockManagerId.hostPort
|
||||||
|
val rddBlocks = sc.getExecutorStorageStatus(a).blocks.size.toString
|
||||||
|
val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString
|
||||||
|
val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString
|
||||||
|
val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString
|
||||||
|
val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0).toString
|
||||||
|
val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0).toString
|
||||||
|
val totalTasks = listener.executorToTaskInfos(a.toString).size.toString
|
||||||
|
|
||||||
|
Seq(
|
||||||
|
execId,
|
||||||
|
hostPort,
|
||||||
|
rddBlocks,
|
||||||
|
memUsed,
|
||||||
|
maxMem,
|
||||||
|
diskUsed,
|
||||||
|
failedTasks,
|
||||||
|
completedTasks,
|
||||||
|
totalTasks
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private[spark] class ExecutorsListener extends SparkListener with Logging {
|
||||||
|
val executorToTasksComplete = HashMap[String, Int]()
|
||||||
|
val executorToTasksFailed = HashMap[String, Int]()
|
||||||
|
val executorToTaskInfos =
|
||||||
|
HashMap[String, ArrayBuffer[(TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])]]()
|
||||||
|
|
||||||
|
override def onTaskEnd(taskEnd: SparkListenerTaskEnd) {
|
||||||
|
val eid = taskEnd.taskInfo.executorId
|
||||||
|
val (failureInfo, metrics): (Option[ExceptionFailure], Option[TaskMetrics]) =
|
||||||
|
taskEnd.reason match {
|
||||||
|
case e: ExceptionFailure =>
|
||||||
|
executorToTasksFailed(eid) = executorToTasksFailed.getOrElse(eid, 0) + 1
|
||||||
|
(Some(e), e.metrics)
|
||||||
|
case _ =>
|
||||||
|
executorToTasksComplete(eid) = executorToTasksComplete.getOrElse(eid, 0) + 1
|
||||||
|
(None, Some(taskEnd.taskMetrics))
|
||||||
|
}
|
||||||
|
val taskList = executorToTaskInfos.getOrElse(
|
||||||
|
eid, ArrayBuffer[(TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])]())
|
||||||
|
taskList += ((taskEnd.taskInfo, metrics, failureInfo))
|
||||||
|
executorToTaskInfos(eid) = taskList
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -33,10 +33,6 @@ private[spark] class IndexPage(parent: BlockManagerUI) {
|
||||||
def render(request: HttpServletRequest): Seq[Node] = {
|
def render(request: HttpServletRequest): Seq[Node] = {
|
||||||
val storageStatusList = sc.getExecutorStorageStatus
|
val storageStatusList = sc.getExecutorStorageStatus
|
||||||
// Calculate macro-level statistics
|
// Calculate macro-level statistics
|
||||||
val maxMem = storageStatusList.map(_.maxMem).reduce(_+_)
|
|
||||||
val remainingMem = storageStatusList.map(_.memRemaining).reduce(_+_)
|
|
||||||
val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize))
|
|
||||||
.reduceOption(_+_).getOrElse(0L)
|
|
||||||
|
|
||||||
val rddHeaders = Seq(
|
val rddHeaders = Seq(
|
||||||
"RDD Name",
|
"RDD Name",
|
||||||
|
@ -46,19 +42,7 @@ private[spark] class IndexPage(parent: BlockManagerUI) {
|
||||||
"Size in Memory",
|
"Size in Memory",
|
||||||
"Size on Disk")
|
"Size on Disk")
|
||||||
val rdds = StorageUtils.rddInfoFromStorageStatus(storageStatusList, sc)
|
val rdds = StorageUtils.rddInfoFromStorageStatus(storageStatusList, sc)
|
||||||
val rddTable = listingTable(rddHeaders, rddRow, rdds)
|
val content = listingTable(rddHeaders, rddRow, rdds)
|
||||||
|
|
||||||
val content =
|
|
||||||
<div class="row">
|
|
||||||
<div class="span12">
|
|
||||||
<ul class="unstyled">
|
|
||||||
<li><strong>Memory:</strong>
|
|
||||||
{Utils.memoryBytesToString(maxMem - remainingMem)} Used
|
|
||||||
({Utils.memoryBytesToString(remainingMem)} Available) </li>
|
|
||||||
<li><strong>Disk:</strong> {Utils.memoryBytesToString(diskSpaceUsed)} Used </li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</div> ++ {rddTable};
|
|
||||||
|
|
||||||
headerSparkPage(content, parent.sc, "Spark Storage ", Storage)
|
headerSparkPage(content, parent.sc, "Spark Storage ", Storage)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue