[SPARK-2392] Executors should not start their own HTTP servers
Executors currently start their own unused HTTP file servers. This is because we use the same SparkEnv class for both executors and drivers, and we do not distinguish this case. In the longer term, we should separate out SparkEnv for the driver and SparkEnv for the executors. Author: Andrew Or <andrewor14@gmail.com> Closes #1335 from andrewor14/executor-http-server and squashes the following commits: 46ef263 [Andrew Or] Start HTTP server only on the driver
This commit is contained in:
parent
e6f7bfcfbf
commit
bf04a390e4
|
@ -79,7 +79,7 @@ class SparkEnv (
|
|||
|
||||
private[spark] def stop() {
|
||||
pythonWorkers.foreach { case(key, worker) => worker.stop() }
|
||||
httpFileServer.stop()
|
||||
Option(httpFileServer).foreach(_.stop())
|
||||
mapOutputTracker.stop()
|
||||
shuffleManager.stop()
|
||||
broadcastManager.stop()
|
||||
|
@ -228,9 +228,15 @@ object SparkEnv extends Logging {
|
|||
|
||||
val cacheManager = new CacheManager(blockManager)
|
||||
|
||||
val httpFileServer = new HttpFileServer(securityManager)
|
||||
httpFileServer.initialize()
|
||||
conf.set("spark.fileserver.uri", httpFileServer.serverUri)
|
||||
val httpFileServer =
|
||||
if (isDriver) {
|
||||
val server = new HttpFileServer(securityManager)
|
||||
server.initialize()
|
||||
conf.set("spark.fileserver.uri", server.serverUri)
|
||||
server
|
||||
} else {
|
||||
null
|
||||
}
|
||||
|
||||
val metricsSystem = if (isDriver) {
|
||||
MetricsSystem.createMetricsSystem("driver", conf, securityManager)
|
||||
|
|
Loading…
Reference in a new issue