5d3e917d09
Adds required ActorSystem argument to BlockManager constructors.
131 lines
4.4 KiB
Scala
131 lines
4.4 KiB
Scala
package spark
|
|
|
|
import akka.actor.ActorSystem
|
|
import akka.actor.ActorSystemImpl
|
|
import akka.remote.RemoteActorRefProvider
|
|
|
|
import serializer.Serializer
|
|
import spark.broadcast.BroadcastManager
|
|
import spark.storage.BlockManager
|
|
import spark.storage.BlockManagerMaster
|
|
import spark.network.ConnectionManager
|
|
import spark.util.AkkaUtils
|
|
|
|
/**
|
|
* Holds all the runtime environment objects for a running Spark instance (either master or worker),
|
|
* including the serializer, Akka actor system, block manager, map output tracker, etc. Currently
|
|
* Spark code finds the SparkEnv through a thread-local variable, so each thread that accesses these
|
|
* objects needs to have the right SparkEnv set. You can get the current environment with
|
|
* SparkEnv.get (e.g. after creating a SparkContext) and set it with SparkEnv.set.
|
|
*/
|
|
class SparkEnv (
|
|
val actorSystem: ActorSystem,
|
|
val serializer: Serializer,
|
|
val closureSerializer: Serializer,
|
|
val cacheTracker: CacheTracker,
|
|
val mapOutputTracker: MapOutputTracker,
|
|
val shuffleFetcher: ShuffleFetcher,
|
|
val broadcastManager: BroadcastManager,
|
|
val blockManager: BlockManager,
|
|
val connectionManager: ConnectionManager,
|
|
val httpFileServer: HttpFileServer
|
|
) {
|
|
|
|
/** No-parameter constructor for unit tests. */
|
|
def this() = {
|
|
this(null, new JavaSerializer, new JavaSerializer, null, null, null, null, null, null, null)
|
|
}
|
|
|
|
def stop() {
|
|
httpFileServer.stop()
|
|
mapOutputTracker.stop()
|
|
cacheTracker.stop()
|
|
shuffleFetcher.stop()
|
|
broadcastManager.stop()
|
|
blockManager.stop()
|
|
blockManager.master.stop()
|
|
actorSystem.shutdown()
|
|
// Unfortunately Akka's awaitTermination doesn't actually wait for the Netty server to shut
|
|
// down, but let's call it anyway in case it gets fixed in a later release
|
|
actorSystem.awaitTermination()
|
|
}
|
|
}
|
|
|
|
object SparkEnv extends Logging {
|
|
private val env = new ThreadLocal[SparkEnv]
|
|
|
|
def set(e: SparkEnv) {
|
|
env.set(e)
|
|
}
|
|
|
|
def get: SparkEnv = {
|
|
env.get()
|
|
}
|
|
|
|
def createFromSystemProperties(
|
|
hostname: String,
|
|
port: Int,
|
|
isMaster: Boolean,
|
|
isLocal: Boolean
|
|
) : SparkEnv = {
|
|
val (actorSystem, boundPort) = AkkaUtils.createActorSystem("spark", hostname, port)
|
|
|
|
// Bit of a hack: If this is the master and our port was 0 (meaning bind to any free port),
|
|
// figure out which port number Akka actually bound to and set spark.master.port to it.
|
|
if (isMaster && port == 0) {
|
|
System.setProperty("spark.master.port", boundPort.toString)
|
|
}
|
|
|
|
val classLoader = Thread.currentThread.getContextClassLoader
|
|
|
|
// Create an instance of the class named by the given Java system property, or by
|
|
// defaultClassName if the property is not set, and return it as a T
|
|
def instantiateClass[T](propertyName: String, defaultClassName: String): T = {
|
|
val name = System.getProperty(propertyName, defaultClassName)
|
|
Class.forName(name, true, classLoader).newInstance().asInstanceOf[T]
|
|
}
|
|
|
|
val serializer = instantiateClass[Serializer]("spark.serializer", "spark.JavaSerializer")
|
|
|
|
val blockManagerMaster = new BlockManagerMaster(actorSystem, isMaster, isLocal)
|
|
val blockManager = new BlockManager(actorSystem, blockManagerMaster, serializer)
|
|
|
|
val connectionManager = blockManager.connectionManager
|
|
|
|
val broadcastManager = new BroadcastManager(isMaster)
|
|
|
|
val closureSerializer = instantiateClass[Serializer](
|
|
"spark.closure.serializer", "spark.JavaSerializer")
|
|
|
|
val cacheTracker = new CacheTracker(actorSystem, isMaster, blockManager)
|
|
blockManager.cacheTracker = cacheTracker
|
|
|
|
val mapOutputTracker = new MapOutputTracker(actorSystem, isMaster)
|
|
|
|
val shuffleFetcher = instantiateClass[ShuffleFetcher](
|
|
"spark.shuffle.fetcher", "spark.BlockStoreShuffleFetcher")
|
|
|
|
val httpFileServer = new HttpFileServer()
|
|
httpFileServer.initialize()
|
|
System.setProperty("spark.fileserver.uri", httpFileServer.serverUri)
|
|
|
|
// Warn about deprecated spark.cache.class property
|
|
if (System.getProperty("spark.cache.class") != null) {
|
|
logWarning("The spark.cache.class property is no longer being used! Specify storage " +
|
|
"levels using the RDD.persist() method instead.")
|
|
}
|
|
|
|
new SparkEnv(
|
|
actorSystem,
|
|
serializer,
|
|
closureSerializer,
|
|
cacheTracker,
|
|
mapOutputTracker,
|
|
shuffleFetcher,
|
|
broadcastManager,
|
|
blockManager,
|
|
connectionManager,
|
|
httpFileServer)
|
|
}
|
|
}
|