[SPARK-18016][SQL][FOLLOWUP] merge declareAddedFunctions, initNestedClasses and declareNestedClasses

## What changes were proposed in this pull request?

These 3 methods have to be used together, so it makes more sense to merge them into one method and then the caller side only need to call one method.

## How was this patch tested?

existing tests.

Author: Wenchen Fan <wenchen@databricks.com>

Closes #18579 from cloud-fan/minor.
This commit is contained in:
Wenchen Fan 2017-07-09 16:30:35 -07:00 committed by gatorsmile
parent 08e0d033b4
commit 680b33f166
8 changed files with 18 additions and 46 deletions

View file

@ -302,29 +302,20 @@ class CodegenContext {
}
/**
* Instantiates all nested, private sub-classes as objects to the `OuterClass`
* Declares all function code. If the added functions are too many, split them into nested
* sub-classes to avoid hitting Java compiler constant pool limitation.
*/
private[sql] def initNestedClasses(): String = {
def declareAddedFunctions(): String = {
val inlinedFunctions = classFunctions(outerClassName).values
// Nested, private sub-classes have no mutable state (though they do reference the outer class'
// mutable state), so we declare and initialize them inline to the OuterClass.
classes.filter(_._1 != outerClassName).map {
val initNestedClasses = classes.filter(_._1 != outerClassName).map {
case (className, classInstance) =>
s"private $className $classInstance = new $className();"
}.mkString("\n")
}
/**
* Declares all function code that should be inlined to the `OuterClass`.
*/
private[sql] def declareAddedFunctions(): String = {
classFunctions(outerClassName).values.mkString("\n")
}
/**
* Declares all nested, private sub-classes and the function code that should be inlined to them.
*/
private[sql] def declareNestedClasses(): String = {
classFunctions.filterKeys(_ != outerClassName).map {
val declareNestedClasses = classFunctions.filterKeys(_ != outerClassName).map {
case (className, functions) =>
s"""
|private class $className {
@ -332,7 +323,9 @@ class CodegenContext {
|}
""".stripMargin
}
}.mkString("\n")
(inlinedFunctions ++ initNestedClasses ++ declareNestedClasses).mkString("\n")
}
final val JAVA_BOOLEAN = "boolean"
final val JAVA_BYTE = "byte"

View file

@ -115,8 +115,6 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP
${ctx.initPartition()}
}
${ctx.declareAddedFunctions()}
public ${classOf[BaseMutableProjection].getName} target(InternalRow row) {
mutableRow = row;
return this;
@ -136,8 +134,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP
return mutableRow;
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}
"""

View file

@ -173,15 +173,12 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
${ctx.initMutableStates()}
}
${ctx.declareAddedFunctions()}
public int compare(InternalRow a, InternalRow b) {
$comparisons
return 0;
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(

View file

@ -66,15 +66,12 @@ object GeneratePredicate extends CodeGenerator[Expression, Predicate] {
${ctx.initPartition()}
}
${ctx.declareAddedFunctions()}
public boolean eval(InternalRow ${ctx.INPUT_ROW}) {
${eval.code}
return !${eval.isNull} && ${eval.value};
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(

View file

@ -175,16 +175,13 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
${ctx.initPartition()}
}
${ctx.declareAddedFunctions()}
public java.lang.Object apply(java.lang.Object _i) {
InternalRow ${ctx.INPUT_ROW} = (InternalRow) _i;
$allExpressions
return mutableRow;
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}
"""

View file

@ -391,8 +391,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
${ctx.initPartition()}
}
${ctx.declareAddedFunctions()}
// Scala.Function1 need this
public java.lang.Object apply(java.lang.Object row) {
return apply((InternalRow) row);
@ -403,8 +401,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
return ${eval.value};
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}
"""

View file

@ -352,14 +352,11 @@ case class WholeStageCodegenExec(child: SparkPlan) extends UnaryExecNode with Co
${ctx.initPartition()}
}
${ctx.declareAddedFunctions()}
protected void processNext() throws java.io.IOException {
${code.trim}
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}
""".trim

View file

@ -192,8 +192,6 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
this.columnIndexes = columnIndexes;
}
${ctx.declareAddedFunctions()}
public boolean hasNext() {
if (currentRow < numRowsInBatch) {
return true;
@ -222,8 +220,7 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
return unsafeRow;
}
${ctx.initNestedClasses()}
${ctx.declareNestedClasses()}
${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(