ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Revert "[SPARK-29046][SQL] Fix NPE in SQLConf.get when active SparkContext is stopping"

Browse source 
This reverts commit 850833fa17.
This commit is contained in:
Dongjoon Hyun 2019-09-14 00:09:45 -07:00
parent 174d9104cf
commit 13b77e52d2
2 changed files with 1 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file

@ -139,8 +139,7 @@ object SQLConf {
}
} else {
val isSchedulerEventLoopThread = SparkContext.getActive
.flatMap { sc => Option(sc.dagScheduler) }
.map(_.eventProcessLoop.eventThread)
.map(_.dagScheduler.eventProcessLoop.eventThread)
.exists(_.getId == Thread.currentThread().getId)
if (isSchedulerEventLoopThread) {
// DAGScheduler event loop thread does not have an active SparkSession, the `confGetter`

19
sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
View file

@ -19,7 +19,6 @@ package org.apache.spark.sql.internal
import org.apache.hadoop.fs.Path
import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext}
import org.apache.spark.sql._
import org.apache.spark.sql.internal.StaticSQLConf._
import org.apache.spark.sql.test.{SharedSparkSession, TestSQLContext}
@ -321,22 +320,4 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
assert(e2.getMessage.contains("spark.sql.shuffle.partitions"))
}
test("SPARK-29046: SQLConf.get shouldn't throw NPE when active SparkContext is stopping") {
// Logically, there's only one case SQLConf.get throws NPE: there's active SparkContext,
// but SparkContext is stopping - especially it sets dagScheduler as null.
val oldSparkContext = SparkContext.getActive
Utils.tryWithSafeFinally {
// this is necessary to set new SparkContext as active: it cleans up active SparkContext
oldSparkContext.foreach(_ => SparkContext.clearActiveContext())
val conf = new SparkConf().setAppName("test").setMaster("local")
LocalSparkContext.withSpark(new SparkContext(conf)) { sc =>
sc.dagScheduler = null
SQLConf.get
}
} {
oldSparkContext.orElse(Some(null)).foreach(SparkContext.setActiveContext)
}
}
}