Merge branch 'spark-633' of github.com:rxin/spark into spark-633
This commit is contained in:
commit
06f855c24d
|
@ -829,7 +829,10 @@ class BlockManager(
|
|||
diskStore.putBytes(blockId, bytes, level)
|
||||
}
|
||||
}
|
||||
memoryStore.remove(blockId)
|
||||
val blockWasRemoved = memoryStore.remove(blockId)
|
||||
if (!blockWasRemoved) {
|
||||
logWarning("Block " + blockId + " could not be dropped from memory as it does not exist")
|
||||
}
|
||||
if (info.tellMaster) {
|
||||
reportBlockStatus(blockId)
|
||||
}
|
||||
|
@ -853,8 +856,12 @@ class BlockManager(
|
|||
val info = blockInfo.get(blockId).orNull
|
||||
if (info != null) info.synchronized {
|
||||
// Removals are idempotent in disk store and memory store. At worst, we get a warning.
|
||||
memoryStore.remove(blockId)
|
||||
diskStore.remove(blockId)
|
||||
val removedFromMemory = memoryStore.remove(blockId)
|
||||
val removedFromDisk = diskStore.remove(blockId)
|
||||
if (!removedFromMemory && !removedFromDisk) {
|
||||
logWarning("Block " + blockId + " could not be removed as it was not found in either " +
|
||||
"the disk or memory store")
|
||||
}
|
||||
blockInfo.remove(blockId)
|
||||
} else {
|
||||
// The block has already been removed; do nothing.
|
||||
|
|
|
@ -31,7 +31,12 @@ abstract class BlockStore(val blockManager: BlockManager) extends Logging {
|
|||
|
||||
def getValues(blockId: String): Option[Iterator[Any]]
|
||||
|
||||
def remove(blockId: String)
|
||||
/**
|
||||
* Remove a block, if it exists.
|
||||
* @param blockId the block to remove.
|
||||
* @return True if the block was found and removed, False otherwise.
|
||||
*/
|
||||
def remove(blockId: String): Boolean
|
||||
|
||||
def contains(blockId: String): Boolean
|
||||
|
||||
|
|
|
@ -92,10 +92,13 @@ private class DiskStore(blockManager: BlockManager, rootDirs: String)
|
|||
getBytes(blockId).map(bytes => blockManager.dataDeserialize(blockId, bytes))
|
||||
}
|
||||
|
||||
override def remove(blockId: String) {
|
||||
override def remove(blockId: String): Boolean = {
|
||||
val file = getFile(blockId)
|
||||
if (file.exists()) {
|
||||
file.delete()
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -90,7 +90,7 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
|
|||
}
|
||||
}
|
||||
|
||||
override def remove(blockId: String) {
|
||||
override def remove(blockId: String): Boolean = {
|
||||
entries.synchronized {
|
||||
val entry = entries.get(blockId)
|
||||
if (entry != null) {
|
||||
|
@ -98,8 +98,9 @@ private class MemoryStore(blockManager: BlockManager, maxMemory: Long)
|
|||
currentMemory -= entry.size
|
||||
logInfo("Block %s of size %d dropped from memory (free %d)".format(
|
||||
blockId, entry.size, freeMemory))
|
||||
true
|
||||
} else {
|
||||
logWarning("Block " + blockId + " could not be removed as it does not exist")
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue