2011-05-17 15:41:13 -04:00
|
|
|
package spark
|
|
|
|
|
|
|
|
class SparkEnv (
|
|
|
|
val cache: Cache,
|
|
|
|
val serializer: Serializer,
|
|
|
|
val cacheTracker: CacheTracker,
|
2011-07-09 17:25:56 -04:00
|
|
|
val mapOutputTracker: MapOutputTracker,
|
|
|
|
val shuffleFetcher: ShuffleFetcher
|
2011-05-17 15:41:13 -04:00
|
|
|
)
|
|
|
|
|
|
|
|
object SparkEnv {
|
|
|
|
private val env = new ThreadLocal[SparkEnv]
|
|
|
|
|
|
|
|
def set(e: SparkEnv) {
|
|
|
|
env.set(e)
|
|
|
|
}
|
|
|
|
|
|
|
|
def get: SparkEnv = {
|
|
|
|
env.get()
|
|
|
|
}
|
|
|
|
|
|
|
|
def createFromSystemProperties(isMaster: Boolean): SparkEnv = {
|
2011-05-22 01:50:08 -04:00
|
|
|
val cacheClass = System.getProperty("spark.cache.class", "spark.SoftReferenceCache")
|
2011-05-17 15:41:13 -04:00
|
|
|
val cache = Class.forName(cacheClass).newInstance().asInstanceOf[Cache]
|
|
|
|
|
2011-07-09 17:25:56 -04:00
|
|
|
val serializerClass = System.getProperty("spark.serializer", "spark.JavaSerializer")
|
|
|
|
val serializer = Class.forName(serializerClass).newInstance().asInstanceOf[Serializer]
|
2011-05-17 15:41:13 -04:00
|
|
|
|
|
|
|
val cacheTracker = new CacheTracker(isMaster, cache)
|
|
|
|
|
|
|
|
val mapOutputTracker = new MapOutputTracker(isMaster)
|
|
|
|
|
2011-07-09 17:25:56 -04:00
|
|
|
val shuffleFetcherClass = System.getProperty("spark.shuffle.fetcher", "spark.SimpleShuffleFetcher")
|
|
|
|
val shuffleFetcher = Class.forName(shuffleFetcherClass).newInstance().asInstanceOf[ShuffleFetcher]
|
|
|
|
|
|
|
|
new SparkEnv(cache, serializer, cacheTracker, mapOutputTracker, shuffleFetcher)
|
2011-05-17 15:41:13 -04:00
|
|
|
}
|
|
|
|
}
|