ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-26851][SQL][FOLLOWUP] Fix cachedColumnBuffers field for Scala 2.11 build

Browse source 
## What changes were proposed in this pull request?

Per https://github.com/apache/spark/pull/23768/files#r259083019 the last change to this line here caused the 2.11 build to fail. It's worked around by making `_cachedColumnBuffers` a field, as it was never set by callers to anything other than its default of null.

## How was this patch tested?

Existing tests.

Closes #23864 from srowen/SPARK-26851.2.

Authored-by: Sean Owen <sean.owen@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
Sean Owen 2019-02-22 15:22:52 +09:00 committed by Takeshi Yamamuro
parent 066379783a
commit 95bb01282c
2 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
View file

@ -213,8 +213,7 @@ class CacheManager extends Logging {
cd.cachedRepresentation.cacheBuilder.clearCache()
val plan = spark.sessionState.executePlan(cd.plan).executedPlan
val newCache = InMemoryRelation(
cacheBuilder = cd.cachedRepresentation
.cacheBuilder.copy(cachedPlan = plan)(_cachedColumnBuffers = null),
cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = plan),
logicalPlan = cd.plan)
val recomputedPlan = cd.copy(cachedRepresentation = newCache)
writeLock {

7
sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
View file

@ -48,8 +48,9 @@ case class CachedRDDBuilder(
batchSize: Int,
storageLevel: StorageLevel,
@transient cachedPlan: SparkPlan,
tableName: Option[String])(
@transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null) {
tableName: Option[String]) {
@transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null
val sizeInBytesStats: LongAccumulator = cachedPlan.sqlContext.sparkContext.longAccumulator
@ -143,7 +144,7 @@ object InMemoryRelation {
child: SparkPlan,
tableName: Option[String],
logicalPlan: LogicalPlan): InMemoryRelation = {
val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)()
val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)
new InMemoryRelation(child.output, cacheBuilder, logicalPlan.outputOrdering)(
statsOfPlanToCache = logicalPlan.stats)
}