ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Delete a file for a block if it already exists.

Browse source
This commit is contained in:
Shivaram Venkataraman 2013-06-03 12:10:00 -07:00
parent 038cfc1a9a
commit a058b0acf3
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
core/src/main/scala/spark/storage/DiskStore.scala
View file

@ -195,9 +195,15 @@ private class DiskStore(blockManager: BlockManager, rootDirs: String)
}
private def createFile(blockId: String, allowAppendExisting: Boolean = false): File = {
val file = getFile(blockId)
var file = getFile(blockId)
if (!allowAppendExisting && file.exists()) {
throw new Exception("File for block " + blockId + " already exists on disk: " + file)
// NOTE(shivaram): Delete the file if it exists. This might happen if a ShuffleMap task
// was rescheduled on the same machine as the old task ?
logWarning("File for block " + blockId + " already exists on disk: " + file + ". Deleting")
file.delete()
// Reopen the file
file = getFile(blockId)
// throw new Exception("File for block " + blockId + " already exists on disk: " + file)
}
file
}