[SPARK-4454] Revert getOrElse() cleanup in DAGScheduler.getCacheLocs()
This method is performance-sensitive and this change wasn't necessary.
This commit is contained in:
parent
d46d6246d2
commit
a51fc7ef9a
|
@ -190,13 +190,15 @@ class DAGScheduler(
|
|||
}
|
||||
|
||||
private def getCacheLocs(rdd: RDD[_]): Array[Seq[TaskLocation]] = cacheLocs.synchronized {
|
||||
cacheLocs.getOrElseUpdate(rdd.id, {
|
||||
// Note: this doesn't use `getOrElse()` because this method is called O(num tasks) times
|
||||
if (!cacheLocs.contains(rdd.id)) {
|
||||
val blockIds = rdd.partitions.indices.map(index => RDDBlockId(rdd.id, index)).toArray[BlockId]
|
||||
val locs = BlockManager.blockIdsToBlockManagers(blockIds, env, blockManagerMaster)
|
||||
blockIds.map { id =>
|
||||
cacheLocs(rdd.id) = blockIds.map { id =>
|
||||
locs.getOrElse(id, Nil).map(bm => TaskLocation(bm.host, bm.executorId))
|
||||
}
|
||||
})
|
||||
}
|
||||
cacheLocs(rdd.id)
|
||||
}
|
||||
|
||||
private def clearCacheLocs(): Unit = cacheLocs.synchronized {
|
||||
|
|
Loading…
Reference in a new issue