From 994f080f8ae3372366e6004600ba791c8a372ff0 Mon Sep 17 00:00:00 2001
From: Matei Zaharia <matei@databricks.com>
Date: Sun, 29 Dec 2013 22:19:33 -0500
Subject: [PATCH] Properly show Spark properties on web UI, and change app name
 property

---
 .../src/main/scala/org/apache/spark/SparkConf.scala |  2 +-
 .../main/scala/org/apache/spark/SparkContext.scala  |  4 ++--
 .../org/apache/spark/ui/env/EnvironmentUI.scala     | 13 ++++++++-----
 .../scala/org/apache/spark/SparkConfSuite.scala     |  2 +-
 python/pyspark/conf.py                              |  2 +-
 python/pyspark/context.py                           |  4 ++--
 6 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 8cecaff5dd..ae52de409e 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -54,7 +54,7 @@ class SparkConf(loadDefaults: Boolean) extends Serializable with Cloneable {
   /** Set a name for your application. Shown in the Spark web UI. */
   def setAppName(name: String): SparkConf = {
     if (name != null) {
-      settings("spark.appName") = name
+      settings("spark.app.name") = name
     }
     this
   }
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 6f54fa7a5a..810ed1860b 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -112,7 +112,7 @@ class SparkContext(
   if (!conf.contains("spark.master")) {
     throw new SparkException("A master URL must be set in your configuration")
   }
-  if (!conf.contains("spark.appName")) {
+  if (!conf.contains("spark.app.name")) {
     throw new SparkException("An application must be set in your configuration")
   }
 
@@ -127,7 +127,7 @@ class SparkContext(
   }
 
   val master = conf.get("spark.master")
-  val appName = conf.get("spark.appName")
+  val appName = conf.get("spark.app.name")
 
   val isLocal = (master == "local" || master.startsWith("local["))
 
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala
index 91fa00a66c..6b4602f928 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentUI.scala
@@ -48,12 +48,15 @@ private[spark] class EnvironmentUI(sc: SparkContext) {
     def jvmTable =
       UIUtils.listingTable(Seq("Name", "Value"), jvmRow, jvmInformation, fixedWidth = true)
 
-    val properties = System.getProperties.iterator.toSeq
-    val classPathProperty = properties.find { case (k, v) =>
-      k.contains("java.class.path")
+    val sparkProperties = sc.conf.getAll.sorted
+
+    val systemProperties = System.getProperties.iterator.toSeq
+    val classPathProperty = systemProperties.find { case (k, v) =>
+      k == "java.class.path"
     }.getOrElse(("", ""))
-    val sparkProperties = properties.filter(_._1.startsWith("spark")).sorted
-    val otherProperties = properties.diff(sparkProperties :+ classPathProperty).sorted
+    val otherProperties = systemProperties.filter { case (k, v) =>
+      k != "java.class.path" && !k.startsWith("spark.")
+    }.sorted
 
     val propertyHeaders = Seq("Name", "Value")
     def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index aaf0b80fe9..77c7b829b3 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -46,7 +46,7 @@ class SparkConfSuite extends FunSuite with LocalSparkContext {
     conf.setExecutorEnv(Seq(("VAR2", "value2"), ("VAR3", "value3")))
 
     assert(conf.get("spark.master") === "local[3]")
-    assert(conf.get("spark.appName") === "My app")
+    assert(conf.get("spark.app.name") === "My app")
     assert(conf.get("spark.home") === "/path")
     assert(conf.get("spark.jars") === "a.jar,b.jar")
     assert(conf.get("spark.executorEnv.VAR1") === "value1")
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
index cf98b0e071..c07dd88307 100644
--- a/python/pyspark/conf.py
+++ b/python/pyspark/conf.py
@@ -23,7 +23,7 @@
 <pyspark.conf.SparkConf object at ...>
 >>> conf.get("spark.master")
 u'local'
->>> conf.get("spark.appName")
+>>> conf.get("spark.app.name")
 u'My app'
 >>> sc = SparkContext(conf=conf)
 >>> sc.master
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 8b028027eb..12ac0299e2 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -104,13 +104,13 @@ class SparkContext(object):
         # Check that we have at least the required parameters
         if not self.conf.contains("spark.master"):
             raise Exception("A master URL must be set in your configuration")
-        if not self.conf.contains("spark.appName"):
+        if not self.conf.contains("spark.app.name"):
             raise Exception("An application name must be set in your configuration")
 
         # Read back our properties from the conf in case we loaded some of them from
         # the classpath or an external config file
         self.master = self.conf.get("spark.master")
-        self.appName = self.conf.get("spark.appName")
+        self.appName = self.conf.get("spark.app.name")
         self.sparkHome = self.conf.getOrElse("spark.home", None)
         for (k, v) in self.conf.getAll():
             if k.startswith("spark.executorEnv."):