[SPARK-14628][CORE][FOLLLOW-UP] Always tracking read/write metrics

## What changes were proposed in this pull request? This PR is a follow up for https://github.com/apache/spark/pull/12417, now we always track input/output/shuffle metrics in spark JSON protocol and status API. Most of the line changes are because of re-generating the gold answer for `HistoryServerSuite`, and we add a lot of 0 values for read/write metrics. ## How was this patch tested? existing tests. Author: Wenchen Fan <wenchen@databricks.com> Closes #12462 from cloud-fan/follow.
2016-04-18 15:17:29 -07:00 · 2016-04-18 15:17:29 -07:00 · 602734084c
parent 6ff0435858
commit 602734084c
40 changed files with 2590 additions and 391 deletions
--- a/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala
@ -58,11 +58,6 @@ class InputMetrics private (_bytesRead: Accumulator[Long], _recordsRead: Accumul
   */
  def recordsRead: Long = _recordsRead.localValue

-  /**
-   * Returns true if this metrics has been updated before.
-   */
-  def isUpdated: Boolean = (bytesRead | recordsRead) != 0
-
  private[spark] def incBytesRead(v: Long): Unit = _bytesRead.add(v)
  private[spark] def incRecordsRead(v: Long): Unit = _recordsRead.add(v)
  private[spark] def setBytesRead(v: Long): Unit = _bytesRead.setValue(v)
--- a/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala
@ -57,11 +57,6 @@ class OutputMetrics private (_bytesWritten: Accumulator[Long], _recordsWritten:
   */
  def recordsWritten: Long = _recordsWritten.localValue

-  /**
-   * Returns true if this metrics has been updated before.
-   */
-  def isUpdated: Boolean = (bytesWritten | recordsWritten) != 0
-
  private[spark] def setBytesWritten(v: Long): Unit = _bytesWritten.setValue(v)
  private[spark] def setRecordsWritten(v: Long): Unit = _recordsWritten.setValue(v)
 }
--- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala
@ -102,11 +102,6 @@ class ShuffleReadMetrics private (
   */
  def totalBlocksFetched: Int = remoteBlocksFetched + localBlocksFetched

-  /**
-   * Returns true if this metrics has been updated before.
-   */
-  def isUpdated: Boolean = (totalBytesRead | totalBlocksFetched | recordsRead | fetchWaitTime) != 0
-
  private[spark] def incRemoteBlocksFetched(v: Int): Unit = _remoteBlocksFetched.add(v)
  private[spark] def incLocalBlocksFetched(v: Int): Unit = _localBlocksFetched.add(v)
  private[spark] def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead.add(v)
--- a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala
@ -68,11 +68,6 @@ class ShuffleWriteMetrics private (
   */
  def writeTime: Long = _writeTime.localValue

-  /**
-   * Returns true if this metrics has been updated before.
-   */
-  def isUpdated: Boolean = (writeTime | recordsWritten | bytesWritten) != 0
-
  private[spark] def incBytesWritten(v: Long): Unit = _bytesWritten.add(v)
  private[spark] def incRecordsWritten(v: Long): Unit = _recordsWritten.add(v)
  private[spark] def incWriteTime(v: Long): Unit = _writeTime.add(v)
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
@ -167,47 +167,32 @@ private[v1] object AllStagesResource {
    // to make it a little easier to deal w/ all of the nested options.  Mostly it lets us just
    // implement one "build" method, which just builds the quantiles for each field.

-    val inputMetrics: Option[InputMetricDistributions] =
+    val inputMetrics: InputMetricDistributions =
      new MetricHelper[InternalInputMetrics, InputMetricDistributions](rawMetrics, quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): Option[InternalInputMetrics] = {
-          if (raw.inputMetrics.isUpdated) {
-            Some(raw.inputMetrics)
-          } else {
-            None
-          }
-        }
+        def getSubmetrics(raw: InternalTaskMetrics): InternalInputMetrics = raw.inputMetrics

        def build: InputMetricDistributions = new InputMetricDistributions(
          bytesRead = submetricQuantiles(_.bytesRead),
          recordsRead = submetricQuantiles(_.recordsRead)
        )
-      }.metricOption
+      }.build

-    val outputMetrics: Option[OutputMetricDistributions] =
+    val outputMetrics: OutputMetricDistributions =
      new MetricHelper[InternalOutputMetrics, OutputMetricDistributions](rawMetrics, quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): Option[InternalOutputMetrics] = {
-          if (raw.outputMetrics.isUpdated) {
-            Some(raw.outputMetrics)
-          } else {
-            None
-          }
-        }
+        def getSubmetrics(raw: InternalTaskMetrics): InternalOutputMetrics = raw.outputMetrics
+
        def build: OutputMetricDistributions = new OutputMetricDistributions(
          bytesWritten = submetricQuantiles(_.bytesWritten),
          recordsWritten = submetricQuantiles(_.recordsWritten)
        )
-      }.metricOption
+      }.build

-    val shuffleReadMetrics: Option[ShuffleReadMetricDistributions] =
+    val shuffleReadMetrics: ShuffleReadMetricDistributions =
      new MetricHelper[InternalShuffleReadMetrics, ShuffleReadMetricDistributions](rawMetrics,
        quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): Option[InternalShuffleReadMetrics] = {
-          if (raw.shuffleReadMetrics.isUpdated) {
-            Some(raw.shuffleReadMetrics)
-          } else {
-            None
-          }
-        }
+        def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleReadMetrics =
+          raw.shuffleReadMetrics
+
        def build: ShuffleReadMetricDistributions = new ShuffleReadMetricDistributions(
          readBytes = submetricQuantiles(_.totalBytesRead),
          readRecords = submetricQuantiles(_.recordsRead),
@ -217,24 +202,20 @@ private[v1] object AllStagesResource {
          totalBlocksFetched = submetricQuantiles(_.totalBlocksFetched),
          fetchWaitTime = submetricQuantiles(_.fetchWaitTime)
        )
-      }.metricOption
+      }.build

-    val shuffleWriteMetrics: Option[ShuffleWriteMetricDistributions] =
+    val shuffleWriteMetrics: ShuffleWriteMetricDistributions =
      new MetricHelper[InternalShuffleWriteMetrics, ShuffleWriteMetricDistributions](rawMetrics,
        quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): Option[InternalShuffleWriteMetrics] = {
-          if (raw.shuffleWriteMetrics.isUpdated) {
-            Some(raw.shuffleWriteMetrics)
-          } else {
-            None
-          }
-        }
+        def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleWriteMetrics =
+          raw.shuffleWriteMetrics
+
        def build: ShuffleWriteMetricDistributions = new ShuffleWriteMetricDistributions(
          writeBytes = submetricQuantiles(_.bytesWritten),
          writeRecords = submetricQuantiles(_.recordsWritten),
          writeTime = submetricQuantiles(_.writeTime)
        )
-      }.metricOption
+      }.build

    new TaskMetricDistributions(
      quantiles = quantiles,
@ -273,84 +254,55 @@ private[v1] object AllStagesResource {
    )
  }

-  def convertInputMetrics(internal: InternalInputMetrics): Option[InputMetrics] = {
-    if (internal.isUpdated) {
-      Some(new InputMetrics(
-        bytesRead = internal.bytesRead,
-        recordsRead = internal.recordsRead
-      ))
-    } else {
-      None
-    }
+  def convertInputMetrics(internal: InternalInputMetrics): InputMetrics = {
+    new InputMetrics(
+      bytesRead = internal.bytesRead,
+      recordsRead = internal.recordsRead
+    )
  }

-  def convertOutputMetrics(internal: InternalOutputMetrics): Option[OutputMetrics] = {
-    if (internal.isUpdated) {
-      Some(new OutputMetrics(
-        bytesWritten = internal.bytesWritten,
-        recordsWritten = internal.recordsWritten
-      ))
-    } else {
-      None
-    }
+  def convertOutputMetrics(internal: InternalOutputMetrics): OutputMetrics = {
+    new OutputMetrics(
+      bytesWritten = internal.bytesWritten,
+      recordsWritten = internal.recordsWritten
+    )
  }

-  def convertShuffleReadMetrics(
-      internal: InternalShuffleReadMetrics): Option[ShuffleReadMetrics] = {
-    if (internal.isUpdated) {
-      Some(new ShuffleReadMetrics(
-        remoteBlocksFetched = internal.remoteBlocksFetched,
-        localBlocksFetched = internal.localBlocksFetched,
-        fetchWaitTime = internal.fetchWaitTime,
-        remoteBytesRead = internal.remoteBytesRead,
-        totalBlocksFetched = internal.totalBlocksFetched,
-        recordsRead = internal.recordsRead
-      ))
-    } else {
-      None
-    }
+  def convertShuffleReadMetrics(internal: InternalShuffleReadMetrics): ShuffleReadMetrics = {
+    new ShuffleReadMetrics(
+      remoteBlocksFetched = internal.remoteBlocksFetched,
+      localBlocksFetched = internal.localBlocksFetched,
+      fetchWaitTime = internal.fetchWaitTime,
+      remoteBytesRead = internal.remoteBytesRead,
+      localBytesRead = internal.localBytesRead,
+      recordsRead = internal.recordsRead
+    )
  }

-  def convertShuffleWriteMetrics(
-      internal: InternalShuffleWriteMetrics): Option[ShuffleWriteMetrics] = {
-    if ((internal.bytesWritten | internal.writeTime | internal.recordsWritten) == 0) {
-      None
-    } else {
-      Some(new ShuffleWriteMetrics(
-        bytesWritten = internal.bytesWritten,
-        writeTime = internal.writeTime,
-        recordsWritten = internal.recordsWritten
-      ))
-    }
+  def convertShuffleWriteMetrics(internal: InternalShuffleWriteMetrics): ShuffleWriteMetrics = {
+    new ShuffleWriteMetrics(
+      bytesWritten = internal.bytesWritten,
+      writeTime = internal.writeTime,
+      recordsWritten = internal.recordsWritten
+    )
  }
 }

 /**
- * Helper for getting distributions from nested metric types.  Many of the metrics we want are
- * contained in options inside TaskMetrics (eg., ShuffleWriteMetrics). This makes it easy to handle
- * the options (returning None if the metrics are all empty), and extract the quantiles for each
- * metric.  After creating an instance, call metricOption to get the result type.
+ * Helper for getting distributions from nested metric types.
 */
 private[v1] abstract class MetricHelper[I, O](
    rawMetrics: Seq[InternalTaskMetrics],
    quantiles: Array[Double]) {

-  def getSubmetrics(raw: InternalTaskMetrics): Option[I]
+  def getSubmetrics(raw: InternalTaskMetrics): I

  def build: O

-  val data: Seq[I] = rawMetrics.flatMap(getSubmetrics)
+  val data: Seq[I] = rawMetrics.map(getSubmetrics)

  /** applies the given function to all input metrics, and returns the quantiles */
  def submetricQuantiles(f: I => Double): IndexedSeq[Double] = {
    Distribution(data.map { d => f(d) }).get.getQuantiles(quantiles)
  }
-
-  def metricOption: Option[O] = {
-    if (data.isEmpty) {
-      None
-    } else {
-      Some(build)
-    }
-  }
 }
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@ -172,10 +172,10 @@ class TaskMetrics private[spark](
    val resultSerializationTime: Long,
    val memoryBytesSpilled: Long,
    val diskBytesSpilled: Long,
-    val inputMetrics: Option[InputMetrics],
-    val outputMetrics: Option[OutputMetrics],
-    val shuffleReadMetrics: Option[ShuffleReadMetrics],
-    val shuffleWriteMetrics: Option[ShuffleWriteMetrics])
+    val inputMetrics: InputMetrics,
+    val outputMetrics: OutputMetrics,
+    val shuffleReadMetrics: ShuffleReadMetrics,
+    val shuffleWriteMetrics: ShuffleWriteMetrics)

 class InputMetrics private[spark](
    val bytesRead: Long,
@ -190,7 +190,7 @@ class ShuffleReadMetrics private[spark](
    val localBlocksFetched: Int,
    val fetchWaitTime: Long,
    val remoteBytesRead: Long,
-    val totalBlocksFetched: Int,
+    val localBytesRead: Long,
    val recordsRead: Long)

 class ShuffleWriteMetrics private[spark](
@ -209,10 +209,10 @@ class TaskMetricDistributions private[spark](
    val memoryBytesSpilled: IndexedSeq[Double],
    val diskBytesSpilled: IndexedSeq[Double],

-    val inputMetrics: Option[InputMetricDistributions],
-    val outputMetrics: Option[OutputMetricDistributions],
-    val shuffleReadMetrics: Option[ShuffleReadMetricDistributions],
-    val shuffleWriteMetrics: Option[ShuffleWriteMetricDistributions])
+    val inputMetrics: InputMetricDistributions,
+    val outputMetrics: OutputMetricDistributions,
+    val shuffleReadMetrics: ShuffleReadMetricDistributions,
+    val shuffleWriteMetrics: ShuffleWriteMetricDistributions)

 class InputMetricDistributions private[spark](
    val bytesRead: IndexedSeq[Double],
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@ -326,39 +326,27 @@ private[spark] object JsonProtocol {
  }

  def taskMetricsToJson(taskMetrics: TaskMetrics): JValue = {
-    val shuffleReadMetrics: JValue = if (taskMetrics.shuffleReadMetrics.isUpdated) {
+    val shuffleReadMetrics: JValue =
      ("Remote Blocks Fetched" -> taskMetrics.shuffleReadMetrics.remoteBlocksFetched) ~
        ("Local Blocks Fetched" -> taskMetrics.shuffleReadMetrics.localBlocksFetched) ~
        ("Fetch Wait Time" -> taskMetrics.shuffleReadMetrics.fetchWaitTime) ~
        ("Remote Bytes Read" -> taskMetrics.shuffleReadMetrics.remoteBytesRead) ~
        ("Local Bytes Read" -> taskMetrics.shuffleReadMetrics.localBytesRead) ~
        ("Total Records Read" -> taskMetrics.shuffleReadMetrics.recordsRead)
-    } else {
-      JNothing
-    }
-    val shuffleWriteMetrics: JValue = if (taskMetrics.shuffleWriteMetrics.isUpdated) {
+    val shuffleWriteMetrics: JValue =
      ("Shuffle Bytes Written" -> taskMetrics.shuffleWriteMetrics.bytesWritten) ~
        ("Shuffle Write Time" -> taskMetrics.shuffleWriteMetrics.writeTime) ~
        ("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten)
-    } else {
-      JNothing
-    }
-    val inputMetrics: JValue = if (taskMetrics.inputMetrics.isUpdated) {
+    val inputMetrics: JValue =
      ("Bytes Read" -> taskMetrics.inputMetrics.bytesRead) ~
        ("Records Read" -> taskMetrics.inputMetrics.recordsRead)
-    } else {
-      JNothing
-    }
-    val outputMetrics: JValue = if (taskMetrics.outputMetrics.isUpdated) {
+    val outputMetrics: JValue =
      ("Bytes Written" -> taskMetrics.outputMetrics.bytesWritten) ~
        ("Records Written" -> taskMetrics.outputMetrics.recordsWritten)
-    } else {
-      JNothing
-    }
    val updatedBlocks =
      JArray(taskMetrics.updatedBlockStatuses.toList.map { case (id, status) =>
        ("Block ID" -> id.toString) ~
-        ("Status" -> blockStatusToJson(status))
+          ("Status" -> blockStatusToJson(status))
      })
    ("Executor Deserialize Time" -> taskMetrics.executorDeserializeTime) ~
    ("Executor Run Time" -> taskMetrics.executorRunTime) ~
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@ -2,108 +2,108 @@
  "id" : "local-1430917381534",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917391398,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:11.398GMT",
    "lastUpdated" : "",
    "duration" : 10505,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917391398,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1430917381535",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917380950,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:00.950GMT",
    "lastUpdated" : "",
    "duration" : 57,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917380950,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1430917380880,
-    "endTimeEpoch" : 1430917380890,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.880GMT",
    "endTime" : "2015-05-06T13:03:00.890GMT",
    "lastUpdated" : "",
    "duration" : 10,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380880,
+    "endTimeEpoch" : 1430917380890,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1426533911241",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1426633910242,
-    "endTimeEpoch" : 1426633945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-17T23:11:50.242GMT",
    "endTime" : "2015-03-17T23:12:25.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426633910242,
+    "endTimeEpoch" : 1426633945177,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1426533910242,
-    "endTimeEpoch" : 1426533945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-16T19:25:10.242GMT",
    "endTime" : "2015-03-16T19:25:45.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426533910242,
+    "endTimeEpoch" : 1426533945177,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1425081759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1425081758277,
-    "endTimeEpoch" : 1425081766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-28T00:02:38.277GMT",
    "endTime" : "2015-02-28T00:02:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1425081758277,
+    "endTimeEpoch" : 1425081766912,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1422981780767",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981779720,
-    "endTimeEpoch" : 1422981788731,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:59.720GMT",
    "endTime" : "2015-02-03T16:43:08.731GMT",
    "lastUpdated" : "",
    "duration" : 9011,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981779720,
+    "endTimeEpoch" : 1422981788731,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1422981759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981758277,
-    "endTimeEpoch" : 1422981766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:38.277GMT",
    "endTime" : "2015-02-03T16:42:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981758277,
+    "endTimeEpoch" : 1422981766912,
+    "lastUpdatedEpoch" : 0
  } ]
 } ]
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@ -2,108 +2,108 @@
  "id" : "local-1430917381534",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917391398,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:11.398GMT",
    "lastUpdated" : "",
    "duration" : 10505,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917391398,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1430917381535",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917380950,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:00.950GMT",
    "lastUpdated" : "",
    "duration" : 57,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917380950,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1430917380880,
-    "endTimeEpoch" : 1430917380890,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.880GMT",
    "endTime" : "2015-05-06T13:03:00.890GMT",
    "lastUpdated" : "",
    "duration" : 10,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380880,
+    "endTimeEpoch" : 1430917380890,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1426533911241",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1426633910242,
-    "endTimeEpoch" : 1426633945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-17T23:11:50.242GMT",
    "endTime" : "2015-03-17T23:12:25.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426633910242,
+    "endTimeEpoch" : 1426633945177,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1426533910242,
-    "endTimeEpoch" : 1426533945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-16T19:25:10.242GMT",
    "endTime" : "2015-03-16T19:25:45.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426533910242,
+    "endTimeEpoch" : 1426533945177,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1425081759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1425081758277,
-    "endTimeEpoch" : 1425081766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-28T00:02:38.277GMT",
    "endTime" : "2015-02-28T00:02:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1425081758277,
+    "endTimeEpoch" : 1425081766912,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1422981780767",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981779720,
-    "endTimeEpoch" : 1422981788731,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:59.720GMT",
    "endTime" : "2015-02-03T16:43:08.731GMT",
    "lastUpdated" : "",
    "duration" : 9011,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981779720,
+    "endTimeEpoch" : 1422981788731,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1422981759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981758277,
-    "endTimeEpoch" : 1422981766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:38.277GMT",
    "endTime" : "2015-02-03T16:42:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981758277,
+    "endTimeEpoch" : 1422981766912,
+    "lastUpdatedEpoch" : 0
  } ]
 } ]
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@ -18,4 +18,4 @@
  "totalShuffleWrite" : 13180,
  "maxMemory" : 278302556,
  "executorLogs" : { }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
@ -12,4 +12,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
@ -12,4 +12,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
@ -40,4 +40,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
@ -2,14 +2,14 @@
  "id" : "local-1422981759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981758277,
-    "endTimeEpoch" : 1422981766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:38.277GMT",
    "endTime" : "2015-02-03T16:42:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981758277,
+    "endTimeEpoch" : 1422981766912,
+    "lastUpdatedEpoch" : 0
  } ]
 } ]
--- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
@ -2,28 +2,28 @@
  "id" : "local-1422981780767",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981779720,
-    "endTimeEpoch" : 1422981788731,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:59.720GMT",
    "endTime" : "2015-02-03T16:43:08.731GMT",
    "lastUpdated" : "",
    "duration" : 9011,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981779720,
+    "endTimeEpoch" : 1422981788731,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1422981759269",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981758277,
-    "endTimeEpoch" : 1422981766912,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:38.277GMT",
    "endTime" : "2015-02-03T16:42:46.912GMT",
    "lastUpdated" : "",
    "duration" : 8635,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981758277,
+    "endTimeEpoch" : 1422981766912,
+    "lastUpdatedEpoch" : 0
  } ]
 } ]
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@ -2,82 +2,80 @@
  "id" : "local-1430917381534",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917391398,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:11.398GMT",
    "lastUpdated" : "",
    "duration" : 10505,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917391398,
+    "lastUpdatedEpoch" : 0
  } ]
-},  {
+}, {
  "id" : "local-1430917381535",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1430917380893,
-    "endTimeEpoch" : 1430917380950,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.893GMT",
    "endTime" : "2015-05-06T13:03:00.950GMT",
    "lastUpdated" : "",
    "duration" : 57,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380893,
+    "endTimeEpoch" : 1430917380950,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1430917380880,
-    "endTimeEpoch" : 1430917380890,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-05-06T13:03:00.880GMT",
    "endTime" : "2015-05-06T13:03:00.890GMT",
    "lastUpdated" : "",
    "duration" : 10,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1430917380880,
+    "endTimeEpoch" : 1430917380890,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
  "id" : "local-1426533911241",
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1426633910242,
-    "endTimeEpoch" : 1426633945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-17T23:11:50.242GMT",
    "endTime" : "2015-03-17T23:12:25.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426633910242,
+    "endTimeEpoch" : 1426633945177,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1426533910242,
-    "endTimeEpoch" : 1426533945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-16T19:25:10.242GMT",
    "endTime" : "2015-03-16T19:25:45.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426533910242,
+    "endTimeEpoch" : 1426533945177,
+    "lastUpdatedEpoch" : 0
  } ]
 }, {
-    "id": "local-1425081759269",
-    "name": "Spark shell",
-    "attempts": [
-      {
-        "startTimeEpoch" : 1425081758277,
-        "endTimeEpoch" : 1425081766912,
-        "lastUpdatedEpoch" : 0,
-        "startTime": "2015-02-28T00:02:38.277GMT",
-        "endTime": "2015-02-28T00:02:46.912GMT",
-        "lastUpdated" : "",
-        "duration" : 8635,
-        "sparkUser": "irashid",
-        "completed": true
-      }
-    ]
+  "id" : "local-1425081759269",
+  "name" : "Spark shell",
+  "attempts" : [ {
+    "startTime" : "2015-02-28T00:02:38.277GMT",
+    "endTime" : "2015-02-28T00:02:46.912GMT",
+    "lastUpdated" : "",
+    "duration" : 8635,
+    "sparkUser" : "irashid",
+    "completed" : true,
+    "startTimeEpoch" : 1425081758277,
+    "endTimeEpoch" : 1425081766912,
+    "lastUpdatedEpoch" : 0
+  } ]
 } ]
--- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
@ -2,14 +2,14 @@
  "id" : "local-1422981780767",
  "name" : "Spark shell",
  "attempts" : [ {
-    "startTimeEpoch" : 1422981779720,
-    "endTimeEpoch" : 1422981788731,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-02-03T16:42:59.720GMT",
    "endTime" : "2015-02-03T16:43:08.731GMT",
    "lastUpdated" : "",
    "duration" : 9011,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1422981779720,
+    "endTimeEpoch" : 1422981788731,
+    "lastUpdatedEpoch" : 0
  } ]
 }
--- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
@ -3,25 +3,25 @@
  "name" : "Spark shell",
  "attempts" : [ {
    "attemptId" : "2",
-    "startTimeEpoch" : 1426633910242,
-    "endTimeEpoch" : 1426633945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-17T23:11:50.242GMT",
    "endTime" : "2015-03-17T23:12:25.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426633910242,
+    "endTimeEpoch" : 1426633945177,
+    "lastUpdatedEpoch" : 0
  }, {
    "attemptId" : "1",
-    "startTimeEpoch" : 1426533910242,
-    "endTimeEpoch" : 1426533945177,
-    "lastUpdatedEpoch" : 0,
    "startTime" : "2015-03-16T19:25:10.242GMT",
    "endTime" : "2015-03-16T19:25:45.177GMT",
    "lastUpdated" : "",
    "duration" : 34935,
    "sparkUser" : "irashid",
-    "completed" : true
+    "completed" : true,
+    "startTimeEpoch" : 1426533910242,
+    "endTimeEpoch" : 1426533945177,
+    "lastUpdatedEpoch" : 0
  } ]
 }
--- a/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
@ -12,4 +12,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-}
+}
--- a/core/src/test/resources/HistoryServerExpectations/one_rdd_storage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_rdd_storage_json_expectation.json
@ -1,64 +0,0 @@
-{
-  "id" : 0,
-  "name" : "0",
-  "numPartitions" : 8,
-  "numCachedPartitions" : 8,
-  "storageLevel" : "Memory Deserialized 1x Replicated",
-  "memoryUsed" : 28000128,
-  "diskUsed" : 0,
-  "dataDistribution" : [ {
-    "address" : "localhost:57971",
-    "memoryUsed" : 28000128,
-    "memoryRemaining" : 250302428,
-    "diskUsed" : 0
-  } ],
-  "partitions" : [ {
-    "blockName" : "rdd_0_0",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_1",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_2",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_3",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_4",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_5",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_6",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  }, {
-    "blockName" : "rdd_0_7",
-    "storageLevel" : "Memory Deserialized 1x Replicated",
-    "memoryUsed" : 3500016,
-    "diskUsed" : 0,
-    "executors" : [ "localhost:57971" ]
-  } ]
-}
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@ -46,6 +46,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 94000,
@ -75,6 +87,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1647,
          "writeTime" : 83000,
@ -104,6 +128,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 88000,
@ -133,6 +169,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 73000,
@ -162,6 +210,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 76000,
@ -191,6 +251,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 98000,
@ -220,6 +292,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1645,
          "writeTime" : 101000,
@ -249,6 +333,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 79000,
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@ -46,6 +46,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 94000,
@ -75,6 +87,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1647,
          "writeTime" : 83000,
@ -104,6 +128,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 88000,
@ -133,6 +169,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 73000,
@ -162,6 +210,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 76000,
@ -191,6 +251,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 98000,
@ -220,6 +292,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1645,
          "writeTime" : 101000,
@ -249,6 +333,18 @@
          "bytesRead" : 3500016,
          "recordsRead" : 0
        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
        "shuffleWriteMetrics" : {
          "bytesWritten" : 1648,
          "writeTime" : 79000,
--- a/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json
@ -1 +1 @@
-[ ]
+[ ]
--- a/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json
@ -1 +1 @@
-[ ]
+[ ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@ -20,6 +20,18 @@
      "bytesRead" : 49294,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3842811,
@ -48,6 +60,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3934399,
@ -76,6 +100,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89885,
@ -104,6 +140,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 1311694,
@ -132,6 +180,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 83022,
@ -160,6 +220,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3675510,
@ -188,6 +260,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 4016617,
@ -216,6 +300,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 2579051,
@ -244,6 +340,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 121551,
@ -272,6 +380,18 @@
      "bytesRead" : 60489,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 101664,
@ -300,6 +420,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94709,
@ -328,6 +460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94507,
@ -356,6 +500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102476,
@ -384,6 +540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95004,
@ -412,6 +580,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95646,
@ -440,6 +620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 602780,
@ -468,6 +660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108320,
@ -496,6 +700,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 99944,
@ -524,6 +740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100836,
@ -552,10 +780,22 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95788,
      "recordsWritten" : 10
    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@ -20,7 +20,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 1,
@ -44,7 +65,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 2,
@ -68,7 +110,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 3,
@ -92,7 +155,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 4,
@ -116,7 +200,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 1,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 5,
@ -140,7 +245,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 6,
@ -164,7 +290,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 7,
@ -188,6 +335,27 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@ -20,7 +20,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 1,
@ -44,7 +65,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 2,
@ -68,7 +110,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 3,
@ -92,7 +155,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 4,
@ -116,7 +200,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 1,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 5,
@ -140,7 +245,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 6,
@ -164,7 +290,28 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
 }, {
  "taskId" : 7,
@ -188,6 +335,27 @@
    "jvmGcTime" : 0,
    "resultSerializationTime" : 2,
    "memoryBytesSpilled" : 0,
-    "diskBytesSpilled" : 0
+    "diskBytesSpilled" : 0,
+    "inputMetrics" : {
+      "bytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
+    "shuffleWriteMetrics" : {
+      "bytesWritten" : 0,
+      "writeTime" : 0,
+      "recordsWritten" : 0
+    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_woffset_length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_woffset_length_expectation.json
@ -20,6 +20,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94709,
@ -48,6 +60,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94507,
@ -76,6 +100,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102476,
@ -104,6 +140,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95004,
@ -132,6 +180,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95646,
@ -160,6 +220,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 602780,
@ -188,6 +260,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108320,
@ -216,6 +300,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 99944,
@ -244,6 +340,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100836,
@ -272,6 +380,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95788,
@ -300,6 +420,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 97716,
@ -328,6 +460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100270,
@ -356,6 +500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 143427,
@ -384,6 +540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 91844,
@ -412,6 +580,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 157194,
@ -440,6 +620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94134,
@ -468,6 +660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108213,
@ -496,6 +700,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102019,
@ -524,6 +740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 104299,
@ -552,6 +780,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 114938,
@ -580,6 +820,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 119770,
@ -608,6 +860,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 92619,
@ -636,6 +900,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89603,
@ -664,6 +940,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 118329,
@ -692,6 +980,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 127746,
@ -720,6 +1020,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 160963,
@ -748,6 +1060,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 123855,
@ -776,6 +1100,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 111869,
@ -804,6 +1140,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 131158,
@ -832,6 +1180,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98748,
@ -860,6 +1220,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94792,
@ -888,6 +1260,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90765,
@ -916,6 +1300,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 103713,
@ -944,6 +1340,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 171516,
@ -972,6 +1380,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98293,
@ -1000,6 +1420,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 92985,
@ -1028,6 +1460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 113322,
@ -1056,6 +1500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 103015,
@ -1084,6 +1540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 139844,
@ -1112,6 +1580,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94984,
@ -1140,6 +1620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90836,
@ -1168,6 +1660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 96013,
@ -1196,6 +1700,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89664,
@ -1224,6 +1740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 92835,
@ -1252,6 +1780,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90506,
@ -1280,6 +1820,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108309,
@ -1308,6 +1860,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90329,
@ -1336,6 +1900,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 96849,
@ -1364,6 +1940,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 97521,
@ -1392,10 +1980,22 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100753,
      "recordsWritten" : 10
    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@ -20,6 +20,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 4016617,
@ -48,6 +60,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3675510,
@ -76,6 +100,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3934399,
@ -104,6 +140,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 83022,
@ -132,6 +180,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 2579051,
@ -160,6 +220,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 1311694,
@ -188,6 +260,18 @@
      "bytesRead" : 49294,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3842811,
@ -216,6 +300,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89885,
@ -244,6 +340,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 143427,
@ -272,6 +380,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100836,
@ -300,6 +420,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 99944,
@ -328,6 +460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100270,
@ -356,6 +500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108320,
@ -384,6 +540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95788,
@ -412,6 +580,18 @@
      "bytesRead" : 60489,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 101664,
@ -440,6 +620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 97716,
@ -468,6 +660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95646,
@ -496,6 +700,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 121551,
@ -524,6 +740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102476,
@ -552,10 +780,22 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95004,
      "recordsWritten" : 10
    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@ -20,6 +20,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 4016617,
@ -48,6 +60,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3675510,
@ -76,6 +100,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3934399,
@ -104,6 +140,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 83022,
@ -132,6 +180,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 2579051,
@ -160,6 +220,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 1311694,
@ -188,6 +260,18 @@
      "bytesRead" : 49294,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 3842811,
@ -216,6 +300,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89885,
@ -244,6 +340,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 143427,
@ -272,6 +380,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100836,
@ -300,6 +420,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 99944,
@ -328,6 +460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100270,
@ -356,6 +500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 108320,
@ -384,6 +540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95788,
@ -412,6 +580,18 @@
      "bytesRead" : 60489,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 101664,
@ -440,6 +620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 97716,
@ -468,6 +660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95646,
@ -496,6 +700,18 @@
      "bytesRead" : 60488,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 121551,
@ -524,6 +740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102476,
@ -552,10 +780,22 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95004,
      "recordsWritten" : 10
    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@ -20,6 +20,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 94792,
@ -48,6 +60,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 95848,
@ -76,6 +100,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90765,
@ -104,6 +140,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 101750,
@ -132,6 +180,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 97521,
@ -160,6 +220,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 171516,
@ -188,6 +260,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 96849,
@ -216,6 +300,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 100753,
@ -244,6 +340,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 89603,
@ -272,6 +380,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102159,
@ -300,6 +420,18 @@
      "bytesRead" : 70565,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 133964,
@ -328,6 +460,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 102779,
@ -356,6 +500,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98472,
@ -384,6 +540,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98748,
@ -412,6 +580,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 103713,
@ -440,6 +620,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 96013,
@ -468,6 +660,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 90836,
@ -496,6 +700,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 92835,
@ -524,6 +740,18 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98293,
@ -552,10 +780,22 @@
      "bytesRead" : 70564,
      "recordsRead" : 10000
    },
+    "outputMetrics" : {
+      "bytesWritten" : 0,
+      "recordsWritten" : 0
+    },
+    "shuffleReadMetrics" : {
+      "remoteBlocksFetched" : 0,
+      "localBlocksFetched" : 0,
+      "fetchWaitTime" : 0,
+      "remoteBytesRead" : 0,
+      "localBytesRead" : 0,
+      "recordsRead" : 0
+    },
    "shuffleWriteMetrics" : {
      "bytesWritten" : 1710,
      "writeTime" : 98069,
      "recordsWritten" : 10
    }
  }
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@ -11,9 +11,22 @@
    "bytesRead" : [ 60488.0, 70564.0, 70565.0 ],
    "recordsRead" : [ 10000.0, 10000.0, 10000.0 ]
  },
+  "outputMetrics" : {
+    "bytesWritten" : [ 0.0, 0.0, 0.0 ],
+    "recordsWritten" : [ 0.0, 0.0, 0.0 ]
+  },
+  "shuffleReadMetrics" : {
+    "readBytes" : [ 0.0, 0.0, 0.0 ],
+    "readRecords" : [ 0.0, 0.0, 0.0 ],
+    "remoteBlocksFetched" : [ 0.0, 0.0, 0.0 ],
+    "localBlocksFetched" : [ 0.0, 0.0, 0.0 ],
+    "fetchWaitTime" : [ 0.0, 0.0, 0.0 ],
+    "remoteBytesRead" : [ 0.0, 0.0, 0.0 ],
+    "totalBlocksFetched" : [ 0.0, 0.0, 0.0 ]
+  },
  "shuffleWriteMetrics" : {
    "writeBytes" : [ 1710.0, 1710.0, 1710.0 ],
    "writeRecords" : [ 10.0, 10.0, 10.0 ],
    "writeTime" : [ 89437.0, 102159.0, 4016617.0 ]
  }
-}
+}
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
@ -7,6 +7,14 @@
  "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
  "memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
  "diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+  "inputMetrics" : {
+    "bytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "recordsRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+  },
+  "outputMetrics" : {
+    "bytesWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "recordsWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+  },
  "shuffleReadMetrics" : {
    "readBytes" : [ 17100.0, 17100.0, 17100.0, 17100.0, 17100.0 ],
    "readRecords" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ],
@ -15,5 +23,10 @@
    "fetchWaitTime" : [ 0.0, 0.0, 0.0, 1.0, 1.0 ],
    "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
    "totalBlocksFetched" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ]
+  },
+  "shuffleWriteMetrics" : {
+    "writeBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "writeRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "writeTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
  }
-}
+}
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
@ -11,9 +11,22 @@
    "bytesRead" : [ 60488.0, 70564.0, 70564.0, 70564.0, 70564.0 ],
    "recordsRead" : [ 10000.0, 10000.0, 10000.0, 10000.0, 10000.0 ]
  },
+  "outputMetrics" : {
+    "bytesWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "recordsWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+  },
+  "shuffleReadMetrics" : {
+    "readBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "readRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "remoteBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "localBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "fetchWaitTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+    "totalBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+  },
  "shuffleWriteMetrics" : {
    "writeBytes" : [ 1710.0, 1710.0, 1710.0, 1710.0, 1710.0 ],
    "writeRecords" : [ 10.0, 10.0, 10.0, 10.0, 10.0 ],
    "writeTime" : [ 90329.0, 95848.0, 102159.0, 121551.0, 2579051.0 ]
  }
-}
+}
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@ -50,7 +50,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "5" : {
@ -75,7 +96,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "4" : {
@ -100,7 +142,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 1,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "7" : {
@ -125,7 +188,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "1" : {
@ -150,7 +234,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "3" : {
@ -175,7 +280,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "6" : {
@ -200,7 +326,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    },
    "0" : {
@ -225,7 +372,28 @@
        "jvmGcTime" : 0,
        "resultSerializationTime" : 2,
        "memoryBytesSpilled" : 0,
-        "diskBytesSpilled" : 0
+        "diskBytesSpilled" : 0,
+        "inputMetrics" : {
+          "bytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "outputMetrics" : {
+          "bytesWritten" : 0,
+          "recordsWritten" : 0
+        },
+        "shuffleReadMetrics" : {
+          "remoteBlocksFetched" : 0,
+          "localBlocksFetched" : 0,
+          "fetchWaitTime" : 0,
+          "remoteBytesRead" : 0,
+          "localBytesRead" : 0,
+          "recordsRead" : 0
+        },
+        "shuffleWriteMetrics" : {
+          "bytesWritten" : 0,
+          "writeTime" : 0,
+          "recordsWritten" : 0
+        }
      }
    }
  },
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
@ -40,4 +40,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-} ]
+} ]
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
@ -26,4 +26,4 @@
  "numCompletedStages" : 1,
  "numSkippedStages" : 0,
  "numFailedStages" : 0
-} ]
+} ]
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@ -153,37 +153,39 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
      code should be (HttpServletResponse.SC_OK)
      jsonOpt should be ('defined)
      errOpt should be (None)
-      val jsonOrg = jsonOpt.get
-
-      // SPARK-10873 added the lastUpdated field for each application's attempt,
-      // the REST API returns the last modified time of EVENT LOG file for this field.
-      // It is not applicable to hard-code this dynamic field in a static expected file,
-      // so here we skip checking the lastUpdated field's value (setting it as "").
-      val json = if (jsonOrg.indexOf("lastUpdated") >= 0) {
-        val subStrings = jsonOrg.split(",")
-        for (i <- subStrings.indices) {
-          if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) {
-            subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0")
-          } else if (subStrings(i).indexOf("lastUpdated") >= 0) {
-            subStrings(i) = "\"lastUpdated\":\"\""
-          }
-        }
-        subStrings.mkString(",")
-      } else {
-        jsonOrg
-      }

      val exp = IOUtils.toString(new FileInputStream(
        new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")))
      // compare the ASTs so formatting differences don't cause failures
      import org.json4s._
      import org.json4s.jackson.JsonMethods._
-      val jsonAst = parse(json)
+      val jsonAst = parse(clearLastUpdated(jsonOpt.get))
      val expAst = parse(exp)
      assertValidDataInJson(jsonAst, expAst)
    }
  }

+  // SPARK-10873 added the lastUpdated field for each application's attempt,
+  // the REST API returns the last modified time of EVENT LOG file for this field.
+  // It is not applicable to hard-code this dynamic field in a static expected file,
+  // so here we skip checking the lastUpdated field's value (setting it as "").
+  private def clearLastUpdated(json: String): String = {
+    if (json.indexOf("lastUpdated") >= 0) {
+      val subStrings = json.split(",")
+      for (i <- subStrings.indices) {
+        if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) {
+          subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0")
+        } else if (subStrings(i).indexOf("lastUpdated") >= 0) {
+          val regex = "\"lastUpdated\"\\s*:\\s*\".*\"".r
+          subStrings(i) = regex.replaceAllIn(subStrings(i), "\"lastUpdated\" : \"\"")
+        }
+      }
+      subStrings.mkString(",")
+    } else {
+      json
+    }
+  }
+
  test("download all logs for app with multiple attempts") {
    doDownloadTest("local-1430917381535", None)
  }
@ -486,7 +488,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
    val json = getUrl(path)
    val file = new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")
    val out = new FileWriter(file)
-    out.write(json)
+    out.write(clearLastUpdated(json))
+    out.write('\n')
    out.close()
  }

--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@ -1111,6 +1111,14 @@ private[spark] object JsonProtocolSuite extends Assertions {
      |      "Shuffle Write Time": 1500,
      |      "Shuffle Records Written": 12
      |    },
+      |    "Input Metrics" : {
+      |      "Bytes Read" : 0,
+      |      "Records Read" : 0
+      |    },
+      |    "Output Metrics" : {
+      |      "Bytes Written" : 0,
+      |      "Records Written" : 0
+      |    },
      |    "Updated Blocks": [
      |      {
      |        "Block ID": "rdd_0_0",
@ -1187,6 +1195,14 @@ private[spark] object JsonProtocolSuite extends Assertions {
      |    "Result Serialization Time": 700,
      |    "Memory Bytes Spilled": 800,
      |    "Disk Bytes Spilled": 0,
+      |    "Shuffle Read Metrics" : {
+      |      "Remote Blocks Fetched" : 0,
+      |      "Local Blocks Fetched" : 0,
+      |      "Fetch Wait Time" : 0,
+      |      "Remote Bytes Read" : 0,
+      |      "Local Bytes Read" : 0,
+      |      "Total Records Read" : 0
+      |    },
      |    "Shuffle Write Metrics": {
      |      "Shuffle Bytes Written": 1200,
      |      "Shuffle Write Time": 1500,
@ -1196,6 +1212,10 @@ private[spark] object JsonProtocolSuite extends Assertions {
      |      "Bytes Read": 2100,
      |      "Records Read": 21
      |    },
+      |     "Output Metrics" : {
+      |      "Bytes Written" : 0,
+      |      "Records Written" : 0
+      |    },
      |    "Updated Blocks": [
      |      {
      |        "Block ID": "rdd_0_0",
@ -1272,6 +1292,19 @@ private[spark] object JsonProtocolSuite extends Assertions {
      |    "Result Serialization Time": 700,
      |    "Memory Bytes Spilled": 800,
      |    "Disk Bytes Spilled": 0,
+      |    "Shuffle Read Metrics" : {
+      |      "Remote Blocks Fetched" : 0,
+      |      "Local Blocks Fetched" : 0,
+      |      "Fetch Wait Time" : 0,
+      |      "Remote Bytes Read" : 0,
+      |      "Local Bytes Read" : 0,
+      |      "Total Records Read" : 0
+      |    },
+      |    "Shuffle Write Metrics" : {
+      |      "Shuffle Bytes Written" : 0,
+      |      "Shuffle Write Time" : 0,
+      |      "Shuffle Records Written" : 0
+      |    },
      |    "Input Metrics": {
      |      "Bytes Read": 2100,
      |      "Records Read": 21
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@ -634,6 +634,20 @@ object MimaExcludes {
        // [SPARK-14628] Simplify task metrics by always tracking read/write metrics
        ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.InputMetrics.readMethod"),
        ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.OutputMetrics.writeMethod")
+      ) ++ Seq(
+        // SPARK-14628: Always track input/output/shuffle metrics
+        ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.totalBlocksFetched"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.this"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.inputMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.outputMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.shuffleWriteMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.shuffleReadMetrics"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.this"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.inputMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.outputMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.shuffleWriteMetrics"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.shuffleReadMetrics"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.this")
      )
    case v if v.startsWith("1.6") =>
      Seq(