[SPARK-17672] Spark 2.0 history server web Ui takes too long for a single application
Added a new API getApplicationInfo(appId: String) in class ApplicationHistoryProvider and class SparkUI to get app info. In this change, FsHistoryProvider can directly fetch one app info in O(1) time complexity compared to O(n) before the change which used an Iterator.find() interface. Both ApplicationCache and OneApplicationResource classes adopt this new api. manual tests Author: Gang Wu <wgtmac@uber.com> Closes #15247 from wgtmac/SPARK-17671.
This commit is contained in:
parent
7f779e7439
commit
cb87b3ced9
|
@ -109,4 +109,9 @@ private[history] abstract class ApplicationHistoryProvider {
|
|||
@throws(classOf[SparkException])
|
||||
def writeEventLogs(appId: String, attemptId: Option[String], zipStream: ZipOutputStream): Unit
|
||||
|
||||
/**
|
||||
* @return the [[ApplicationHistoryInfo]] for the appId if it exists.
|
||||
*/
|
||||
def getApplicationInfo(appId: String): Option[ApplicationHistoryInfo]
|
||||
|
||||
}
|
||||
|
|
|
@ -224,6 +224,10 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
|
|||
|
||||
override def getListing(): Iterable[FsApplicationHistoryInfo] = applications.values
|
||||
|
||||
override def getApplicationInfo(appId: String): Option[FsApplicationHistoryInfo] = {
|
||||
applications.get(appId)
|
||||
}
|
||||
|
||||
override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
|
||||
try {
|
||||
applications.get(appId).flatMap { appInfo =>
|
||||
|
|
|
@ -182,6 +182,10 @@ class HistoryServer(
|
|||
getApplicationList().iterator.map(ApplicationsListResource.appHistoryInfoToPublicAppInfo)
|
||||
}
|
||||
|
||||
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
|
||||
provider.getApplicationInfo(appId).map(ApplicationsListResource.appHistoryInfoToPublicAppInfo)
|
||||
}
|
||||
|
||||
override def writeEventLogs(
|
||||
appId: String,
|
||||
attemptId: Option[String],
|
||||
|
|
|
@ -222,6 +222,7 @@ private[spark] object ApiRootResource {
|
|||
private[spark] trait UIRoot {
|
||||
def getSparkUI(appKey: String): Option[SparkUI]
|
||||
def getApplicationInfoList: Iterator[ApplicationInfo]
|
||||
def getApplicationInfo(appId: String): Option[ApplicationInfo]
|
||||
|
||||
/**
|
||||
* Write the event logs for the given app to the [[ZipOutputStream]] instance. If attemptId is
|
||||
|
|
|
@ -24,7 +24,7 @@ private[v1] class OneApplicationResource(uiRoot: UIRoot) {
|
|||
|
||||
@GET
|
||||
def getApp(@PathParam("appId") appId: String): ApplicationInfo = {
|
||||
val apps = uiRoot.getApplicationInfoList.find { _.id == appId }
|
||||
val apps = uiRoot.getApplicationInfo(appId)
|
||||
apps.getOrElse(throw new NotFoundException("unknown app: " + appId))
|
||||
}
|
||||
|
||||
|
|
|
@ -126,6 +126,10 @@ private[spark] class SparkUI private (
|
|||
))
|
||||
))
|
||||
}
|
||||
|
||||
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
|
||||
getApplicationInfoList.find(_.id == appId)
|
||||
}
|
||||
}
|
||||
|
||||
private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String)
|
||||
|
|
Loading…
Reference in a new issue