[SPARK-1199][REPL] Remove VALId and use the original import style for defined classes.
This is an alternate solution to #1176. Author: Prashant Sharma <prashant.s@imaginea.com> Closes #1179 from ScrapCodes/SPARK-1199/repl-fix-second-approach and squashes the following commits: 820b34b [Prashant Sharma] Here we generate two kinds of import wrappers based on whether it is a class or not.
This commit is contained in:
parent
544880457d
commit
d43415075b
|
@ -744,7 +744,7 @@ import org.apache.spark.util.Utils
|
||||||
*
|
*
|
||||||
* Read! Eval! Print! Some of that not yet centralized here.
|
* Read! Eval! Print! Some of that not yet centralized here.
|
||||||
*/
|
*/
|
||||||
class ReadEvalPrint(lineId: Int) {
|
class ReadEvalPrint(val lineId: Int) {
|
||||||
def this() = this(freshLineId())
|
def this() = this(freshLineId())
|
||||||
|
|
||||||
private var lastRun: Run = _
|
private var lastRun: Run = _
|
||||||
|
@ -1241,7 +1241,10 @@ import org.apache.spark.util.Utils
|
||||||
// old style
|
// old style
|
||||||
beSilentDuring(parse(code)) foreach { ts =>
|
beSilentDuring(parse(code)) foreach { ts =>
|
||||||
ts foreach { t =>
|
ts foreach { t =>
|
||||||
withoutUnwrapping(logDebug(asCompactString(t)))
|
if (isShow || isShowRaw)
|
||||||
|
withoutUnwrapping(echo(asCompactString(t)))
|
||||||
|
else
|
||||||
|
withoutUnwrapping(logDebug(asCompactString(t)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -182,15 +182,26 @@ trait SparkImports {
|
||||||
// ambiguity errors will not be generated. Also, quote
|
// ambiguity errors will not be generated. Also, quote
|
||||||
// the name of the variable, so that we don't need to
|
// the name of the variable, so that we don't need to
|
||||||
// handle quoting keywords separately.
|
// handle quoting keywords separately.
|
||||||
|
case x: ClassHandler =>
|
||||||
|
// I am trying to guess if the import is a defined class
|
||||||
|
// This is an ugly hack, I am not 100% sure of the consequences.
|
||||||
|
// Here we, let everything but "defined classes" use the import with val.
|
||||||
|
// The reason for this is, otherwise the remote executor tries to pull the
|
||||||
|
// classes involved and may fail.
|
||||||
|
for (imv <- x.definedNames) {
|
||||||
|
val objName = req.lineRep.readPath
|
||||||
|
code.append("import " + objName + ".INSTANCE" + req.accessPath + ".`" + imv + "`\n")
|
||||||
|
}
|
||||||
|
|
||||||
case x =>
|
case x =>
|
||||||
for (imv <- x.definedNames) {
|
for (imv <- x.definedNames) {
|
||||||
if (currentImps contains imv) addWrapper()
|
if (currentImps contains imv) addWrapper()
|
||||||
val objName = req.lineRep.readPath
|
val objName = req.lineRep.readPath
|
||||||
val valName = "$VAL" + newValId();
|
val valName = "$VAL" + req.lineRep.lineId
|
||||||
|
|
||||||
if(!code.toString.endsWith(".`" + imv + "`;\n")) { // Which means already imported
|
if(!code.toString.endsWith(".`" + imv + "`;\n")) { // Which means already imported
|
||||||
code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
|
code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
|
||||||
code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
|
code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
|
||||||
}
|
}
|
||||||
// code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
|
// code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
|
||||||
// code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
|
// code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
|
||||||
|
@ -211,10 +222,4 @@ trait SparkImports {
|
||||||
private def membersAtPickler(sym: Symbol): List[Symbol] =
|
private def membersAtPickler(sym: Symbol): List[Symbol] =
|
||||||
beforePickler(sym.info.nonPrivateMembers.toList)
|
beforePickler(sym.info.nonPrivateMembers.toList)
|
||||||
|
|
||||||
private var curValId = 0
|
|
||||||
|
|
||||||
private def newValId(): Int = {
|
|
||||||
curValId += 1
|
|
||||||
curValId
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -235,6 +235,18 @@ class ReplSuite extends FunSuite {
|
||||||
assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output)
|
assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test("SPARK-1199-simple-reproduce") {
|
||||||
|
val output = runInterpreter("local-cluster[1,1,512]",
|
||||||
|
"""
|
||||||
|
|case class Sum(exp: String, exp2: String)
|
||||||
|
|val a = Sum("A", "B")
|
||||||
|
|def b(a: Sum): String = a match { case Sum(_, _) => "Found Sum" }
|
||||||
|
|b(a)
|
||||||
|
""".stripMargin)
|
||||||
|
assertDoesNotContain("error:", output)
|
||||||
|
assertDoesNotContain("Exception", output)
|
||||||
|
}
|
||||||
|
|
||||||
if (System.getenv("MESOS_NATIVE_LIBRARY") != null) {
|
if (System.getenv("MESOS_NATIVE_LIBRARY") != null) {
|
||||||
test("running on Mesos") {
|
test("running on Mesos") {
|
||||||
val output = runInterpreter("localquiet",
|
val output = runInterpreter("localquiet",
|
||||||
|
|
Loading…
Reference in a new issue