ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[MINOR][CORE] Fix nits in MetricsSystemSuite

Browse source 
## What changes were proposed in this pull request?
Fixing nits in MetricsSystemSuite file
1) Using Sink instead of Source while casting
2) Using meaningful naming for variables, which reflect their usage

## How was this patch tested?
Ran the tests locally and all of them passing

Author: Srinivasa Reddy Vundela <vsr@cloudera.com>

Closes #19699 from vundela/master.
This commit is contained in:
Srinivasa Reddy Vundela 2017-11-09 09:53:41 -08:00 committed by Marcelo Vanzin
parent 77f74539ec
commit 6b19c0735d
1 changed files with 19 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
core/src/test/scala/org/apache/spark/metrics/MetricsSystemSuite.scala
View file

@ -25,6 +25,7 @@ import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.master.MasterSource
import org.apache.spark.internal.config._
import org.apache.spark.metrics.sink.Sink
import org.apache.spark.metrics.source.{Source, StaticSources}
class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateMethodTester{
@ -42,7 +43,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
val metricsSystem = MetricsSystem.createMetricsSystem("default", conf, securityMgr)
metricsSystem.start()
val sources = PrivateMethod[ArrayBuffer[Source]]('sources)
val sinks = PrivateMethod[ArrayBuffer[Source]]('sinks)
val sinks = PrivateMethod[ArrayBuffer[Sink]]('sinks)
assert(metricsSystem.invokePrivate(sources()).length === StaticSources.allSources.length)
assert(metricsSystem.invokePrivate(sinks()).length === 0)
@ -53,7 +54,7 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
val metricsSystem = MetricsSystem.createMetricsSystem("test", conf, securityMgr)
metricsSystem.start()
val sources = PrivateMethod[ArrayBuffer[Source]]('sources)
val sinks = PrivateMethod[ArrayBuffer[Source]]('sinks)
val sinks = PrivateMethod[ArrayBuffer[Sink]]('sinks)
assert(metricsSystem.invokePrivate(sources()).length === StaticSources.allSources.length)
assert(metricsSystem.invokePrivate(sinks()).length === 1)
@ -126,9 +127,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set("spark.executor.id", executorId)
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
assert(metricName === s"$appId.$executorId.${source.sourceName}")
}
@ -142,9 +143,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set("spark.executor.id", executorId)
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
assert(metricName === source.sourceName)
}
@ -158,9 +159,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set("spark.app.id", appId)
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
assert(metricName === source.sourceName)
}
@ -176,9 +177,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set("spark.executor.id", executorId)
val instanceName = "testInstance"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = testMetricsSystem.buildRegistryName(source)
// Even if spark.app.id and spark.executor.id are set, they are not used for the metric name.
assert(metricName != s"$appId.$executorId.${source.sourceName}")
@ -200,9 +201,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set(METRICS_NAMESPACE, "${spark.app.name}")
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
assert(metricName === s"$appName.$executorId.${source.sourceName}")
}
@ -218,9 +219,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set(METRICS_NAMESPACE, namespaceToResolve)
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
// If the user set the spark.metrics.namespace property to an expansion of another property
// (say ${spark.doesnotexist}, the unresolved name (i.e. literally ${spark.doesnotexist})
// is used as the root logger name.
@ -238,9 +239,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set(METRICS_NAMESPACE, "${spark.app.name}")
val instanceName = "executor"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val executorMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = executorMetricsSystem.buildRegistryName(source)
assert(metricName === source.sourceName)
}
@ -259,9 +260,9 @@ class MetricsSystemSuite extends SparkFunSuite with BeforeAndAfter with PrivateM
conf.set("spark.executor.id", executorId)
val instanceName = "testInstance"
val driverMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val testMetricsSystem = MetricsSystem.createMetricsSystem(instanceName, conf, securityMgr)
val metricName = driverMetricsSystem.buildRegistryName(source)
val metricName = testMetricsSystem.buildRegistryName(source)
// Even if spark.app.id and spark.executor.id are set, they are not used for the metric name.
assert(metricName != s"$appId.$executorId.${source.sourceName}")