From 95bb01282cc94f95bbc69aafcbc1550b137238be Mon Sep 17 00:00:00 2001 From: Sean Owen Date: Fri, 22 Feb 2019 15:22:52 +0900 Subject: [PATCH] [SPARK-26851][SQL][FOLLOWUP] Fix cachedColumnBuffers field for Scala 2.11 build ## What changes were proposed in this pull request? Per https://github.com/apache/spark/pull/23768/files#r259083019 the last change to this line here caused the 2.11 build to fail. It's worked around by making `_cachedColumnBuffers` a field, as it was never set by callers to anything other than its default of null. ## How was this patch tested? Existing tests. Closes #23864 from srowen/SPARK-26851.2. Authored-by: Sean Owen Signed-off-by: Takeshi Yamamuro --- .../org/apache/spark/sql/execution/CacheManager.scala | 3 +-- .../spark/sql/execution/columnar/InMemoryRelation.scala | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala index c6ee7354d0..f7a78ea0cb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala @@ -213,8 +213,7 @@ class CacheManager extends Logging { cd.cachedRepresentation.cacheBuilder.clearCache() val plan = spark.sessionState.executePlan(cd.plan).executedPlan val newCache = InMemoryRelation( - cacheBuilder = cd.cachedRepresentation - .cacheBuilder.copy(cachedPlan = plan)(_cachedColumnBuffers = null), + cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = plan), logicalPlan = cd.plan) val recomputedPlan = cd.copy(cachedRepresentation = newCache) writeLock { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala index bc6e958f3e..7180853e4c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala @@ -48,8 +48,9 @@ case class CachedRDDBuilder( batchSize: Int, storageLevel: StorageLevel, @transient cachedPlan: SparkPlan, - tableName: Option[String])( - @transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null) { + tableName: Option[String]) { + + @transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null val sizeInBytesStats: LongAccumulator = cachedPlan.sqlContext.sparkContext.longAccumulator @@ -143,7 +144,7 @@ object InMemoryRelation { child: SparkPlan, tableName: Option[String], logicalPlan: LogicalPlan): InMemoryRelation = { - val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)() + val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName) new InMemoryRelation(child.output, cacheBuilder, logicalPlan.outputOrdering)( statsOfPlanToCache = logicalPlan.stats) }