[SPARK-26851][SQL][FOLLOWUP] Fix cachedColumnBuffers field for Scala 2.11 build
## What changes were proposed in this pull request? Per https://github.com/apache/spark/pull/23768/files#r259083019 the last change to this line here caused the 2.11 build to fail. It's worked around by making `_cachedColumnBuffers` a field, as it was never set by callers to anything other than its default of null. ## How was this patch tested? Existing tests. Closes #23864 from srowen/SPARK-26851.2. Authored-by: Sean Owen <sean.owen@databricks.com> Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
parent
066379783a
commit
95bb01282c
|
@ -213,8 +213,7 @@ class CacheManager extends Logging {
|
|||
cd.cachedRepresentation.cacheBuilder.clearCache()
|
||||
val plan = spark.sessionState.executePlan(cd.plan).executedPlan
|
||||
val newCache = InMemoryRelation(
|
||||
cacheBuilder = cd.cachedRepresentation
|
||||
.cacheBuilder.copy(cachedPlan = plan)(_cachedColumnBuffers = null),
|
||||
cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = plan),
|
||||
logicalPlan = cd.plan)
|
||||
val recomputedPlan = cd.copy(cachedRepresentation = newCache)
|
||||
writeLock {
|
||||
|
|
|
@ -48,8 +48,9 @@ case class CachedRDDBuilder(
|
|||
batchSize: Int,
|
||||
storageLevel: StorageLevel,
|
||||
@transient cachedPlan: SparkPlan,
|
||||
tableName: Option[String])(
|
||||
@transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null) {
|
||||
tableName: Option[String]) {
|
||||
|
||||
@transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null
|
||||
|
||||
val sizeInBytesStats: LongAccumulator = cachedPlan.sqlContext.sparkContext.longAccumulator
|
||||
|
||||
|
@ -143,7 +144,7 @@ object InMemoryRelation {
|
|||
child: SparkPlan,
|
||||
tableName: Option[String],
|
||||
logicalPlan: LogicalPlan): InMemoryRelation = {
|
||||
val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)()
|
||||
val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)
|
||||
new InMemoryRelation(child.output, cacheBuilder, logicalPlan.outputOrdering)(
|
||||
statsOfPlanToCache = logicalPlan.stats)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue