ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #103 from amplab/optimizations

Browse source 
Optimizations cherry-picked from SIGMOD branches
This commit is contained in:
Reynold Xin 2013-12-15 17:25:33 -08:00
parent 8a56c1ff67 bad85b051d
commit ded10ce5b0
2 changed files with 35 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala
View file

@ -249,8 +249,7 @@ class OpenHashSet[@specialized(Long, Int) T: ClassManifest](
* in the lower bits, similar to java.util.HashMap * in the lower bits, similar to java.util.HashMap
*/ */
private def hashcode(h: Int): Int = { private def hashcode(h: Int): Int = {
val r = h ^ (h >>> 20) ^ (h >>> 12) it.unimi.dsi.fastutil.HashCommon.murmurHash3(h)
r ^ (r >>> 7) ^ (r >>> 4)
} }
private def nextPowerOf2(n: Int): Int = { private def nextPowerOf2(n: Int): Int = {

44
graph/src/main/scala/org/apache/spark/graph/impl/Serializers.scala
View file

@ -295,7 +295,7 @@ abstract class ShuffleDeserializationStream(s: InputStream) extends Deserializat
var i: Int = 0 var i: Int = 0
def readOrThrow(): Int = { def readOrThrow(): Int = {
val in = s.read() val in = s.read()
if (in < 0) throw new java.io.EOFException if (in < 0) throw new EOFException
in & 0xFF in & 0xFF
} }
var b: Int = readOrThrow() var b: Int = readOrThrow()
@ -309,21 +309,45 @@ abstract class ShuffleDeserializationStream(s: InputStream) extends Deserializat
} }
def readVarLong(optimizePositive: Boolean): Long = { def readVarLong(optimizePositive: Boolean): Long = {
// TODO: unroll the while loop.
var value: Long = 0L
def readOrThrow(): Int = { def readOrThrow(): Int = {
val in = s.read() val in = s.read()
if (in < 0) throw new java.io.EOFException if (in < 0) throw new EOFException
in & 0xFF in & 0xFF
} }
var i: Int = 0 var b = readOrThrow()
var b: Int = readOrThrow() var ret: Long = b & 0x7F
while (i < 56 && (b & 0x80) != 0) { if ((b & 0x80) != 0) {
value |= (b & 0x7F).toLong << i
i += 7
b = readOrThrow() b = readOrThrow()
ret |= (b & 0x7F) << 7
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F) << 14
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F) << 21
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F).toLong << 28
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F).toLong << 35
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F).toLong << 42
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= (b & 0x7F).toLong << 49
if ((b & 0x80) != 0) {
b = readOrThrow()
ret |= b.toLong << 56
}
}
}
}
}
}
}
} }
val ret = value | (b.toLong << i)
if (!optimizePositive) (ret >>> 1) ^ -(ret & 1) else ret if (!optimizePositive) (ret >>> 1) ^ -(ret & 1) else ret
} }