2011-05-17 15:41:13 -04:00
|
|
|
package spark
|
|
|
|
|
2012-06-29 02:51:28 -04:00
|
|
|
import akka.actor.ActorSystem
|
2012-09-12 17:50:37 -04:00
|
|
|
import akka.actor.ActorSystemImpl
|
|
|
|
import akka.remote.RemoteActorRefProvider
|
2012-06-07 03:25:47 -04:00
|
|
|
|
2012-08-05 04:10:51 -04:00
|
|
|
import spark.broadcast.BroadcastManager
|
2012-06-07 03:25:47 -04:00
|
|
|
import spark.storage.BlockManager
|
|
|
|
import spark.storage.BlockManagerMaster
|
|
|
|
import spark.network.ConnectionManager
|
2012-06-30 17:45:55 -04:00
|
|
|
import spark.util.AkkaUtils
|
2012-06-07 03:25:47 -04:00
|
|
|
|
2012-09-28 21:28:13 -04:00
|
|
|
/**
|
|
|
|
* Holds all the runtime environment objects for a running Spark instance (either master or worker),
|
|
|
|
* including the serializer, Akka actor system, block manager, map output tracker, etc. Currently
|
|
|
|
* Spark code finds the SparkEnv through a thread-local variable, so each thread that accesses these
|
|
|
|
* objects needs to have the right SparkEnv set. You can get the current environment with
|
|
|
|
* SparkEnv.get (e.g. after creating a SparkContext) and set it with SparkEnv.set.
|
|
|
|
*/
|
2011-05-17 15:41:13 -04:00
|
|
|
class SparkEnv (
|
2012-06-29 02:51:28 -04:00
|
|
|
val actorSystem: ActorSystem,
|
2012-06-07 03:25:47 -04:00
|
|
|
val serializer: Serializer,
|
|
|
|
val closureSerializer: Serializer,
|
|
|
|
val cacheTracker: CacheTracker,
|
|
|
|
val mapOutputTracker: MapOutputTracker,
|
|
|
|
val shuffleFetcher: ShuffleFetcher,
|
2012-08-05 04:10:51 -04:00
|
|
|
val broadcastManager: BroadcastManager,
|
2012-06-07 03:25:47 -04:00
|
|
|
val blockManager: BlockManager,
|
2012-08-30 14:01:43 -04:00
|
|
|
val connectionManager: ConnectionManager,
|
|
|
|
val httpFileServer: HttpFileServer
|
2012-06-07 03:25:47 -04:00
|
|
|
) {
|
|
|
|
|
|
|
|
/** No-parameter constructor for unit tests. */
|
2012-07-30 14:24:01 -04:00
|
|
|
def this() = {
|
2012-09-29 02:40:56 -04:00
|
|
|
this(null, new JavaSerializer, new JavaSerializer, null, null, null, null, null, null, null)
|
2012-07-30 14:24:01 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
def stop() {
|
2012-08-30 14:01:43 -04:00
|
|
|
httpFileServer.stop()
|
2012-07-30 14:24:01 -04:00
|
|
|
mapOutputTracker.stop()
|
|
|
|
cacheTracker.stop()
|
|
|
|
shuffleFetcher.stop()
|
2012-08-05 04:10:51 -04:00
|
|
|
broadcastManager.stop()
|
2012-07-30 14:24:01 -04:00
|
|
|
blockManager.stop()
|
2012-07-30 16:53:47 -04:00
|
|
|
blockManager.master.stop()
|
2012-07-30 14:24:01 -04:00
|
|
|
actorSystem.shutdown()
|
2012-09-24 18:39:58 -04:00
|
|
|
// Akka's awaitTermination doesn't actually wait until the port is unbound, so sleep a bit
|
|
|
|
Thread.sleep(100)
|
2012-07-30 14:24:01 -04:00
|
|
|
actorSystem.awaitTermination()
|
2012-09-12 17:50:37 -04:00
|
|
|
// Akka's awaitTermination doesn't actually wait until the port is unbound, so sleep a bit
|
|
|
|
Thread.sleep(100)
|
2012-07-30 14:24:01 -04:00
|
|
|
}
|
2012-06-07 03:25:47 -04:00
|
|
|
}
|
2011-05-17 15:41:13 -04:00
|
|
|
|
2012-10-06 20:27:14 -04:00
|
|
|
object SparkEnv extends Logging {
|
2011-05-17 15:41:13 -04:00
|
|
|
private val env = new ThreadLocal[SparkEnv]
|
|
|
|
|
|
|
|
def set(e: SparkEnv) {
|
|
|
|
env.set(e)
|
|
|
|
}
|
|
|
|
|
|
|
|
def get: SparkEnv = {
|
|
|
|
env.get()
|
|
|
|
}
|
|
|
|
|
2012-06-29 19:01:36 -04:00
|
|
|
def createFromSystemProperties(
|
|
|
|
hostname: String,
|
|
|
|
port: Int,
|
|
|
|
isMaster: Boolean,
|
|
|
|
isLocal: Boolean
|
|
|
|
) : SparkEnv = {
|
|
|
|
|
2012-06-30 17:45:55 -04:00
|
|
|
val (actorSystem, boundPort) = AkkaUtils.createActorSystem("spark", hostname, port)
|
2012-06-29 02:51:28 -04:00
|
|
|
|
2012-06-29 19:01:36 -04:00
|
|
|
// Bit of a hack: If this is the master and our port was 0 (meaning bind to any free port),
|
|
|
|
// figure out which port number Akka actually bound to and set spark.master.port to it.
|
|
|
|
if (isMaster && port == 0) {
|
2012-06-30 17:45:55 -04:00
|
|
|
System.setProperty("spark.master.port", boundPort.toString)
|
2012-06-29 19:01:36 -04:00
|
|
|
}
|
|
|
|
|
2012-09-28 19:14:05 -04:00
|
|
|
val classLoader = Thread.currentThread.getContextClassLoader
|
|
|
|
|
|
|
|
// Create an instance of the class named by the given Java system property, or by
|
|
|
|
// defaultClassName if the property is not set, and return it as a T
|
|
|
|
def instantiateClass[T](propertyName: String, defaultClassName: String): T = {
|
|
|
|
val name = System.getProperty(propertyName, defaultClassName)
|
|
|
|
Class.forName(name, true, classLoader).newInstance().asInstanceOf[T]
|
|
|
|
}
|
|
|
|
|
|
|
|
val serializer = instantiateClass[Serializer]("spark.serializer", "spark.JavaSerializer")
|
2012-06-07 03:25:47 -04:00
|
|
|
|
2012-07-30 16:53:47 -04:00
|
|
|
val blockManagerMaster = new BlockManagerMaster(actorSystem, isMaster, isLocal)
|
|
|
|
val blockManager = new BlockManager(blockManagerMaster, serializer)
|
2012-06-07 03:25:47 -04:00
|
|
|
|
2012-09-28 21:28:13 -04:00
|
|
|
val connectionManager = blockManager.connectionManager
|
2011-05-17 15:41:13 -04:00
|
|
|
|
2012-08-05 04:10:51 -04:00
|
|
|
val broadcastManager = new BroadcastManager(isMaster)
|
|
|
|
|
2012-09-28 19:14:05 -04:00
|
|
|
val closureSerializer = instantiateClass[Serializer](
|
|
|
|
"spark.closure.serializer", "spark.JavaSerializer")
|
|
|
|
|
2012-06-29 02:51:28 -04:00
|
|
|
val cacheTracker = new CacheTracker(actorSystem, isMaster, blockManager)
|
2012-06-07 03:25:47 -04:00
|
|
|
blockManager.cacheTracker = cacheTracker
|
2011-05-17 15:41:13 -04:00
|
|
|
|
2012-06-29 02:51:28 -04:00
|
|
|
val mapOutputTracker = new MapOutputTracker(actorSystem, isMaster)
|
2011-05-17 15:41:13 -04:00
|
|
|
|
2012-09-28 19:14:05 -04:00
|
|
|
val shuffleFetcher = instantiateClass[ShuffleFetcher](
|
|
|
|
"spark.shuffle.fetcher", "spark.BlockStoreShuffleFetcher")
|
2012-08-30 14:01:43 -04:00
|
|
|
|
|
|
|
val httpFileServer = new HttpFileServer()
|
|
|
|
httpFileServer.initialize()
|
|
|
|
System.setProperty("spark.fileserver.uri", httpFileServer.serverUri)
|
2012-02-10 01:14:19 -05:00
|
|
|
|
2012-10-06 20:27:14 -04:00
|
|
|
// Warn about deprecated spark.cache.class property
|
|
|
|
if (System.getProperty("spark.cache.class") != null) {
|
|
|
|
logWarning("The spark.cache.class property is no longer being used! Specify storage " +
|
|
|
|
"levels using the RDD.persist() method instead.")
|
|
|
|
}
|
|
|
|
|
2012-06-29 02:51:28 -04:00
|
|
|
new SparkEnv(
|
|
|
|
actorSystem,
|
|
|
|
serializer,
|
|
|
|
closureSerializer,
|
|
|
|
cacheTracker,
|
|
|
|
mapOutputTracker,
|
|
|
|
shuffleFetcher,
|
2012-08-05 04:10:51 -04:00
|
|
|
broadcastManager,
|
2012-06-29 02:51:28 -04:00
|
|
|
blockManager,
|
2012-08-30 14:01:43 -04:00
|
|
|
connectionManager,
|
|
|
|
httpFileServer)
|
2011-05-17 15:41:13 -04:00
|
|
|
}
|
|
|
|
}
|