ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

UseHttpPipelining option is brought back in. It works!

Browse source
This commit is contained in:
Mosharaf Chowdhury 2010-12-07 10:07:30 -08:00
parent 7e2d72c328
commit f82cc17bc5
1 changed files with 49 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
src/scala/spark/LocalFileShuffle.scala
View file

@ -68,9 +68,27 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
(myIndex, LocalFileShuffle.serverUri)
}).collect()
val splitsByUri = new ArrayBuffer[(String, Int)]
// Load config option to decide whether or not to use HTTP pipelining
val UseHttpPipelining =
System.getProperty("spark.shuffle.UseHttpPipelining", "true").toBoolean
// Build a traversable list of pairs of server URI and split. Needs to be
// of type TraversableOnce[(String, ArrayBuffer[Int])]
val splitsByUri = if (UseHttpPipelining) {
// Build a hashmap from server URI to list of splits (to facillitate
// fetching all the URIs on a server within a single connection)
val splitsByUriHM = new HashMap[String, ArrayBuffer[Int]]
for ((inputId, serverUri) <- outputLocs) {
splitsByUri += ((serverUri, inputId))
splitsByUriHM.getOrElseUpdate(serverUri, ArrayBuffer()) += inputId
}
splitsByUriHM.toArray
} else {
// Don't use HTTP pipelining
val splitsByUriAB = new ArrayBuffer[(String, ArrayBuffer[Int])]
for ((inputId, serverUri) <- outputLocs) {
splitsByUriAB += ((serverUri, new ArrayBuffer[Int] += inputId))
}
splitsByUriAB.toArray
}
// TODO: Could broadcast splitsByUri
@ -97,12 +115,10 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
val splitIndex = selectRandomSplit
if (splitIndex != -1) {
val (serverUri, inputId) = splitsByUri (splitIndex)
val url =
"%s/shuffle/%d/%d/%d".format(serverUri, shuffleId, inputId, myId)
val (serverUri, inputIds) = splitsByUri (splitIndex)
threadPool.execute (
new ShuffleClient (url, splitIndex, mergeCombiners))
threadPool.execute ( new ShuffleClient (serverUri, shuffleId.toInt,
inputIds, myId, splitIndex, mergeCombiners))
// splitIndex is in transit. Will be unset in the ShuffleClient
splitsInRequestBitVector.synchronized {
@ -138,16 +154,21 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
}
}
class ShuffleClient (url: String, splitIndex: Int,
class ShuffleClient (serverUri: String, shuffleId: Int,
inputIds: ArrayBuffer[Int], myId: Int, splitIndex: Int,
mergeCombiners: (C, C) => C)
extends Thread with Logging {
private var receptionSucceeded = false
override def run: Unit = {
try {
for (inputId <- inputIds) {
val url =
"%s/shuffle/%d/%d/%d".format(serverUri, shuffleId, inputId, myId)
val readStartTime = System.currentTimeMillis
logInfo ("BEGIN READ: " + url)
try {
val inputStream = new ObjectInputStream(new URL(url).openStream())
try {
while (true) {
@ -164,6 +185,11 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
}
inputStream.close()
logInfo ("END READ: " + url)
val readTime = (System.currentTimeMillis - readStartTime)
logInfo ("Reading " + url + " took " + readTime + " millis.")
}
// Reception completed. Update stats.
hasSplitsBitVector.synchronized {
hasSplitsBitVector.set (splitIndex)
@ -176,10 +202,6 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
}
receptionSucceeded = true
logInfo ("END READ: " + url)
val readTime = (System.currentTimeMillis - readStartTime)
logInfo ("Reading " + url + " took " + readTime + " millis.")
} catch {
// EOFException is expected to happen because sender can break
// connection due to timeout