[SPARK-4313][WebUI][Yarn] Fix link issue of the executor thread dump page in yarn-cluster mode
In yarn-cluster mode, the Web UI is running behind a yarn proxy server. Some features(or bugs?) of yarn proxy server will break the links for thread dump. 1. Yarn proxy server will do http redirect internally, so if opening `http://example.com:8088/cluster/app/application_1415344371838_0012/executors`, it will fetch `http://example.com:8088/cluster/app/application_1415344371838_0012/executors/` and return the content but won't change the link in the browser. Then when a user clicks `Thread Dump`, it will jump to `http://example.com:8088/proxy/application_1415344371838_0012/threadDump/?executorId=2`. This is a wrong link. The correct link should be `http://example.com:8088/proxy/application_1415344371838_0012/executors/threadDump/?executorId=2`. Adding "/" to the tab links will fix it. 2. Yarn proxy server has a bug about the URL encode/decode. When a user accesses `http://example.com:8088/proxy/application_1415344371838_0006/executors/threadDump/?executorId=%3Cdriver%3E`, the yarn proxy server will require `http://example.com:36429/executors/threadDump/?executorId=%25253Cdriver%25253E`. But Spark web server expects `http://example.com:36429/executors/threadDump/?executorId=%3Cdriver%3E`. Related to [YARN-2844](https://issues.apache.org/jira/browse/YARN-2844). For now, it's a tricky approach to bypass the yarn bug. ![threaddump](https://cloud.githubusercontent.com/assets/1000778/4972567/d1ccba64-68ad-11e4-983e-257530cef35a.png) Author: zsxwing <zsxwing@gmail.com> Closes #3183 from zsxwing/SPARK-4313 and squashes the following commits: 3379ca8 [zsxwing] Encode the executor id in the thread dump link and update the comment abfa063 [zsxwing] Fix link issue of the executor thread dump page in yarn-cluster mode
This commit is contained in:
parent
5c265ccde0
commit
156cf3333d
|
@ -175,7 +175,7 @@ private[spark] object UIUtils extends Logging {
|
|||
val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..."
|
||||
val header = activeTab.headerTabs.map { tab =>
|
||||
<li class={if (tab == activeTab) "active" else ""}>
|
||||
<a href={prependBaseUri(activeTab.basePath, "/" + tab.prefix)}>{tab.name}</a>
|
||||
<a href={prependBaseUri(activeTab.basePath, "/" + tab.prefix + "/")}>{tab.name}</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.spark.ui.exec
|
||||
|
||||
import java.net.URLDecoder
|
||||
import javax.servlet.http.HttpServletRequest
|
||||
|
||||
import scala.util.Try
|
||||
|
@ -29,7 +30,19 @@ private[ui] class ExecutorThreadDumpPage(parent: ExecutorsTab) extends WebUIPage
|
|||
private val sc = parent.sc
|
||||
|
||||
def render(request: HttpServletRequest): Seq[Node] = {
|
||||
val executorId = Option(request.getParameter("executorId")).getOrElse {
|
||||
val executorId = Option(request.getParameter("executorId")).map {
|
||||
executorId =>
|
||||
// Due to YARN-2844, "<driver>" in the url will be encoded to "%25253Cdriver%25253E" when
|
||||
// running in yarn-cluster mode. `request.getParameter("executorId")` will return
|
||||
// "%253Cdriver%253E". Therefore we need to decode it until we get the real id.
|
||||
var id = executorId
|
||||
var decodedId = URLDecoder.decode(id, "UTF-8")
|
||||
while (id != decodedId) {
|
||||
id = decodedId
|
||||
decodedId = URLDecoder.decode(id, "UTF-8")
|
||||
}
|
||||
id
|
||||
}.getOrElse {
|
||||
return Text(s"Missing executorId parameter")
|
||||
}
|
||||
val time = System.currentTimeMillis()
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.spark.ui.exec
|
||||
|
||||
import java.net.URLEncoder
|
||||
import javax.servlet.http.HttpServletRequest
|
||||
|
||||
import scala.xml.Node
|
||||
|
@ -139,8 +140,9 @@ private[ui] class ExecutorsPage(
|
|||
</td>
|
||||
{
|
||||
if (threadDumpEnabled) {
|
||||
val encodedId = URLEncoder.encode(info.id, "UTF-8")
|
||||
<td>
|
||||
<a href={s"threadDump/?executorId=${info.id}"}>Thread Dump</a>
|
||||
<a href={s"threadDump/?executorId=${encodedId}"}>Thread Dump</a>
|
||||
</td>
|
||||
} else {
|
||||
Seq.empty
|
||||
|
|
Loading…
Reference in a new issue