Revert "[SPARK-35253][SPARK-35398][SQL][BUILD] Bump up the janino version to v3.1.4"
### What changes were proposed in this pull request?
This PR reverts https://github.com/apache/spark/pull/32455 and its followup https://github.com/apache/spark/pull/32536 , because the new janino version has a bug that is not fixed yet: https://github.com/janino-compiler/janino/pull/148
### Why are the changes needed?
avoid regressions
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
existing tests
Closes #33302 from cloud-fan/revert.
Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
(cherry picked from commit ae6199af44
)
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
This commit is contained in:
parent
eae79dd31b
commit
c1d8ccfb64
|
@ -37,7 +37,7 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
|
|||
commons-cli/1.2//commons-cli-1.2.jar
|
||||
commons-codec/1.15//commons-codec-1.15.jar
|
||||
commons-collections/3.2.2//commons-collections-3.2.2.jar
|
||||
commons-compiler/3.1.4//commons-compiler-3.1.4.jar
|
||||
commons-compiler/3.0.16//commons-compiler-3.0.16.jar
|
||||
commons-compress/1.20//commons-compress-1.20.jar
|
||||
commons-configuration/1.6//commons-configuration-1.6.jar
|
||||
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
|
||||
|
@ -122,7 +122,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
|
|||
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
|
||||
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
|
||||
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
|
||||
janino/3.1.4//janino-3.1.4.jar
|
||||
janino/3.0.16//janino-3.0.16.jar
|
||||
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
|
||||
javax.inject/1//javax.inject-1.jar
|
||||
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
|
||||
|
|
|
@ -31,7 +31,7 @@ chill_2.12/0.10.0//chill_2.12-0.10.0.jar
|
|||
commons-cli/1.2//commons-cli-1.2.jar
|
||||
commons-codec/1.15//commons-codec-1.15.jar
|
||||
commons-collections/3.2.2//commons-collections-3.2.2.jar
|
||||
commons-compiler/3.1.4//commons-compiler-3.1.4.jar
|
||||
commons-compiler/3.0.16//commons-compiler-3.0.16.jar
|
||||
commons-compress/1.20//commons-compress-1.20.jar
|
||||
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
|
||||
commons-dbcp/1.4//commons-dbcp-1.4.jar
|
||||
|
@ -98,7 +98,7 @@ jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
|
|||
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
|
||||
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
|
||||
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
|
||||
janino/3.1.4//janino-3.1.4.jar
|
||||
janino/3.0.16//janino-3.0.16.jar
|
||||
javassist/3.25.0-GA//javassist-3.25.0-GA.jar
|
||||
javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
|
||||
javolution/5.5.1//javolution-5.5.1.jar
|
||||
|
|
2
pom.xml
2
pom.xml
|
@ -182,7 +182,7 @@
|
|||
<commons-pool2.version>2.6.2</commons-pool2.version>
|
||||
<datanucleus-core.version>4.1.17</datanucleus-core.version>
|
||||
<guava.version>14.0.1</guava.version>
|
||||
<janino.version>3.1.4</janino.version>
|
||||
<janino.version>3.0.16</janino.version>
|
||||
<jersey.version>2.34</jersey.version>
|
||||
<joda.version>2.10.10</joda.version>
|
||||
<jodd.version>3.5.2</jodd.version>
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.spark.sql.catalyst.expressions.codegen
|
||||
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.util.{Map => JavaMap}
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.collection.mutable
|
||||
|
@ -26,8 +27,8 @@ import scala.util.control.NonFatal
|
|||
|
||||
import com.google.common.cache.{CacheBuilder, CacheLoader}
|
||||
import com.google.common.util.concurrent.{ExecutionError, UncheckedExecutionException}
|
||||
import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
|
||||
import org.codehaus.janino.ClassBodyEvaluator
|
||||
import org.codehaus.commons.compiler.CompileException
|
||||
import org.codehaus.janino.{ByteArrayClassLoader, ClassBodyEvaluator, InternalCompilerException, SimpleCompiler}
|
||||
import org.codehaus.janino.util.ClassFile
|
||||
|
||||
import org.apache.spark.{TaskContext, TaskKilledException}
|
||||
|
@ -1519,7 +1520,14 @@ object CodeGenerator extends Logging {
|
|||
*/
|
||||
private def updateAndGetCompilationStats(evaluator: ClassBodyEvaluator): ByteCodeStats = {
|
||||
// First retrieve the generated classes.
|
||||
val classes = evaluator.getBytecodes.asScala
|
||||
val classes = {
|
||||
val resultField = classOf[SimpleCompiler].getDeclaredField("result")
|
||||
resultField.setAccessible(true)
|
||||
val loader = resultField.get(evaluator).asInstanceOf[ByteArrayClassLoader]
|
||||
val classesField = loader.getClass.getDeclaredField("classes")
|
||||
classesField.setAccessible(true)
|
||||
classesField.get(loader).asInstanceOf[JavaMap[String, Array[Byte]]].asScala
|
||||
}
|
||||
|
||||
// Then walk the classes to get at the method bytecode.
|
||||
val codeAttr = Utils.classForName("org.codehaus.janino.util.ClassFile$CodeAttribute")
|
||||
|
|
|
@ -29,7 +29,8 @@ import java.util.concurrent.TimeoutException
|
|||
import com.fasterxml.jackson.core.JsonToken
|
||||
import org.apache.hadoop.fs.{FileAlreadyExistsException, FileStatus, Path}
|
||||
import org.apache.hadoop.fs.permission.FsPermission
|
||||
import org.codehaus.commons.compiler.{CompileException, InternalCompilerException}
|
||||
import org.codehaus.commons.compiler.CompileException
|
||||
import org.codehaus.janino.InternalCompilerException
|
||||
|
||||
import org.apache.spark.{Partition, SparkArithmeticException, SparkException, SparkUpgradeException}
|
||||
import org.apache.spark.executor.CommitDeniedException
|
||||
|
|
|
@ -563,8 +563,8 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
|
|||
assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
|
||||
}
|
||||
|
||||
// TODO (SPARK-35579): Fix this bug in janino or work around it in Spark.
|
||||
ignore("SPARK-35578: final local variable bug in janino") {
|
||||
// TODO (SPARK-35579): Fix this bug in janino and upgrade janino in Spark.
|
||||
test("SPARK-35578: final local variable bug in janino") {
|
||||
val code =
|
||||
"""
|
||||
|public Object generate(Object[] references) {
|
||||
|
|
Loading…
Reference in a new issue