ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-22854][UI] Read Spark version from event logs.

Browse source 
The code was ignoring SparkListenerLogStart, which was added
somewhat recently to record the Spark version used to generate
an event log.

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #20049 from vanzin/SPARK-22854.
This commit is contained in:
Marcelo Vanzin 2017-12-22 09:25:39 +08:00 committed by Wenchen Fan
parent 7ab165b706
commit c0abb1d994
2 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
View file

@ -48,7 +48,7 @@ private[spark] class AppStatusListener(
import config._
private val sparkVersion = SPARK_VERSION
private var sparkVersion = SPARK_VERSION
private var appInfo: v1.ApplicationInfo = null
private var appSummary = new AppSummary(0, 0)
private var coresPerTask: Int = 1
@ -90,6 +90,11 @@ private[spark] class AppStatusListener(
}
}
override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
case SparkListenerLogStart(version) => sparkVersion = version
case _ =>
}
override def onApplicationStart(event: SparkListenerApplicationStart): Unit = {
assert(event.appId.isDefined, "Application without IDs are not supported.")

3
core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
View file

@ -103,6 +103,8 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
test("scheduler events") {
val listener = new AppStatusListener(store, conf, true)
listener.onOtherEvent(SparkListenerLogStart("TestSparkVersion"))
// Start the application.
time += 1
listener.onApplicationStart(SparkListenerApplicationStart(
@ -125,6 +127,7 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
assert(attempt.endTime.getTime() === -1L)
assert(attempt.sparkUser === "user")
assert(!attempt.completed)
assert(attempt.appSparkVersion === "TestSparkVersion")
}
// Start a couple of executors.