Merge branch 'master' into scala-2.9
Conflicts: project/build/SparkProject.scala
This commit is contained in:
commit
beb9c117f0
|
@ -108,7 +108,7 @@ class Executor extends mesos.Executor with Logging {
|
||||||
loader = {
|
loader = {
|
||||||
try {
|
try {
|
||||||
val klass = Class.forName("spark.repl.ExecutorClassLoader").asInstanceOf[Class[_ <: ClassLoader]]
|
val klass = Class.forName("spark.repl.ExecutorClassLoader").asInstanceOf[Class[_ <: ClassLoader]]
|
||||||
val constructor = klass.getConstructor(classUri.getClass, loader.getClass)
|
val constructor = klass.getConstructor(classOf[String], classOf[ClassLoader])
|
||||||
constructor.newInstance(classUri, loader)
|
constructor.newInstance(classUri, loader)
|
||||||
} catch {
|
} catch {
|
||||||
case _: ClassNotFoundException => loader
|
case _: ClassNotFoundException => loader
|
||||||
|
|
|
@ -3,6 +3,7 @@ package spark
|
||||||
import mesos.SlaveOffer
|
import mesos.SlaveOffer
|
||||||
|
|
||||||
import org.apache.hadoop.io.LongWritable
|
import org.apache.hadoop.io.LongWritable
|
||||||
|
import org.apache.hadoop.io.NullWritable
|
||||||
import org.apache.hadoop.io.Text
|
import org.apache.hadoop.io.Text
|
||||||
import org.apache.hadoop.mapred.FileInputFormat
|
import org.apache.hadoop.mapred.FileInputFormat
|
||||||
import org.apache.hadoop.mapred.InputFormat
|
import org.apache.hadoop.mapred.InputFormat
|
||||||
|
@ -51,6 +52,15 @@ extends RDD[(K, V)](sc) {
|
||||||
.asInstanceOf[InputFormat[K, V]]
|
.asInstanceOf[InputFormat[K, V]]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper method for creating a Hadoop Writable, because the commonly used
|
||||||
|
// NullWritable class has no constructor
|
||||||
|
def createWritable[T](clazz: Class[T]): T = {
|
||||||
|
if (clazz == classOf[NullWritable])
|
||||||
|
NullWritable.get().asInstanceOf[T]
|
||||||
|
else
|
||||||
|
clazz.newInstance()
|
||||||
|
}
|
||||||
|
|
||||||
override def splits = splits_
|
override def splits = splits_
|
||||||
|
|
||||||
override def compute(theSplit: Split) = new Iterator[(K, V)] {
|
override def compute(theSplit: Split) = new Iterator[(K, V)] {
|
||||||
|
@ -63,8 +73,8 @@ extends RDD[(K, V)](sc) {
|
||||||
val fmt = createInputFormat(conf)
|
val fmt = createInputFormat(conf)
|
||||||
reader = fmt.getRecordReader(split.inputSplit.value, conf, Reporter.NULL)
|
reader = fmt.getRecordReader(split.inputSplit.value, conf, Reporter.NULL)
|
||||||
|
|
||||||
val key: K = keyClass.newInstance()
|
val key: K = createWritable(keyClass)
|
||||||
val value: V = valueClass.newInstance()
|
val value: V = createWritable(valueClass)
|
||||||
var gotNext = false
|
var gotNext = false
|
||||||
var finished = false
|
var finished = false
|
||||||
|
|
||||||
|
|
|
@ -16,10 +16,11 @@ class SparkProject(info: ProjectInfo) extends ParentProject(info) with IdeaProje
|
||||||
|
|
||||||
lazy val bagel = project("bagel", "Bagel", new BagelProject(_), core)
|
lazy val bagel = project("bagel", "Bagel", new BagelProject(_), core)
|
||||||
|
|
||||||
lazy val jettyWebapp = "org.eclipse.jetty" % "jetty-webapp" % "7.4.1.v20110513" % "provided"
|
|
||||||
|
|
||||||
trait BaseProject extends BasicScalaProject with ScalaPaths with BasicPackagePaths with Eclipsify with IdeaProject {
|
trait BaseProject extends BasicScalaProject with ScalaPaths with BasicPackagePaths with Eclipsify with IdeaProject {
|
||||||
override def compileOptions = super.compileOptions ++ Seq(Unchecked)
|
override def compileOptions = super.compileOptions ++ Seq(Unchecked)
|
||||||
|
|
||||||
|
lazy val jettyServer = "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526"
|
||||||
|
|
||||||
override def packageDocsJar = defaultJarPath("-javadoc.jar")
|
override def packageDocsJar = defaultJarPath("-javadoc.jar")
|
||||||
override def packageSrcJar= defaultJarPath("-sources.jar")
|
override def packageSrcJar= defaultJarPath("-sources.jar")
|
||||||
lazy val sourceArtifact = Artifact.sources(artifactID)
|
lazy val sourceArtifact = Artifact.sources(artifactID)
|
||||||
|
@ -38,20 +39,15 @@ class SparkProject(info: ProjectInfo) extends ParentProject(info) with IdeaProje
|
||||||
val asm = "asm" % "asm-all" % "3.3.1"
|
val asm = "asm" % "asm-all" % "3.3.1"
|
||||||
val scalaTest = "org.scalatest" % "scalatest_2.9.0" % "1.4.1" % "test"
|
val scalaTest = "org.scalatest" % "scalatest_2.9.0" % "1.4.1" % "test"
|
||||||
val scalaCheck = "org.scala-tools.testing" % "scalacheck_2.9.0" % "1.8" % "test"
|
val scalaCheck = "org.scala-tools.testing" % "scalacheck_2.9.0" % "1.8" % "test"
|
||||||
val jetty = jettyWebapp
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class ReplProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport {
|
class ReplProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport
|
||||||
val jetty = jettyWebapp
|
|
||||||
}
|
|
||||||
|
|
||||||
class ExamplesProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject {
|
class ExamplesProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject {
|
||||||
val colt = "colt" % "colt" % "1.2.0"
|
val colt = "colt" % "colt" % "1.2.0"
|
||||||
}
|
}
|
||||||
|
|
||||||
class BagelProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport {
|
class BagelProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport
|
||||||
val jetty = jettyWebapp
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue