ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-25317][CORE] Avoid perf regression in Murmur3 Hash on UTF8String

Browse source 
## What changes were proposed in this pull request?

SPARK-10399 introduced a performance regression on the hash computation for UTF8String.

The regression can be evaluated with the code attached in the JIRA. That code runs in about 120 us per method on my laptop (MacBook Pro 2.5 GHz Intel Core i7, RAM 16 GB 1600 MHz DDR3) while the code from branch 2.3 takes on the same machine about 45 us for me. After the PR, the code takes about 45 us on the master branch too.

## How was this patch tested?

running the perf test from the JIRA

Closes #22338 from mgaido91/SPARK-25317.

Authored-by: Marco Gaido <marcogaido91@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
Marco Gaido 2018-09-06 15:27:59 +08:00 committed by Wenchen Fan
parent d749d034a8
commit 64c314e22f
1 changed files with 14 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java
View file

@ -19,6 +19,7 @@ package org.apache.spark.unsafe.hash;
import com.google.common.primitives.Ints;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.types.UTF8String;
@ -59,7 +60,7 @@ public final class Murmur3_x86_32 {
// This is based on Guava's `Murmur32_Hasher.processRemaining(ByteBuffer)` method.
int lengthInBytes = Ints.checkedCast(base.size());
assert (lengthInBytes % 8 == 0): "lengthInBytes must be a multiple of 8 (word-aligned)";
int h1 = hashBytesByIntBlock(base, seed);
int h1 = hashBytesByIntBlock(base, lengthInBytes, seed);
return fmix(h1, lengthInBytes);
}
@ -69,14 +70,19 @@ public final class Murmur3_x86_32 {
}
public static int hashUnsafeBytesBlock(MemoryBlock base, int seed) {
return hashUnsafeBytesBlock(base, Ints.checkedCast(base.size()), seed);
}
private static int hashUnsafeBytesBlock(MemoryBlock base, int lengthInBytes, int seed) {
// This is not compatible with original and another implementations.
// But remain it for backward compatibility for the components existing before 2.3.
int lengthInBytes = Ints.checkedCast(base.size());
assert (lengthInBytes >= 0): "lengthInBytes cannot be negative";
int lengthAligned = lengthInBytes - lengthInBytes % 4;
int h1 = hashBytesByIntBlock(base.subBlock(0, lengthAligned), seed);
int h1 = hashBytesByIntBlock(base, lengthAligned, seed);
long offset = base.getBaseOffset();
Object o = base.getBaseObject();
for (int i = lengthAligned; i < lengthInBytes; i++) {
int halfWord = base.getByte(i);
int halfWord = Platform.getByte(o, offset + i);
int k1 = mixK1(halfWord);
h1 = mixH1(h1, k1);
}
@ -84,7 +90,7 @@ public final class Murmur3_x86_32 {
}
public static int hashUTF8String(UTF8String str, int seed) {
return hashUnsafeBytesBlock(str.getMemoryBlock(), seed);
return hashUnsafeBytesBlock(str.getMemoryBlock(), str.numBytes(), seed);
}
public static int hashUnsafeBytes(Object base, long offset, int lengthInBytes, int seed) {
@ -101,7 +107,7 @@ public final class Murmur3_x86_32 {
int lengthInBytes = Ints.checkedCast(base.size());
assert (lengthInBytes >= 0) : "lengthInBytes cannot be negative";
int lengthAligned = lengthInBytes - lengthInBytes % 4;
int h1 = hashBytesByIntBlock(base.subBlock(0, lengthAligned), seed);
int h1 = hashBytesByIntBlock(base, lengthAligned, seed);
int k1 = 0;
for (int i = lengthAligned, shift = 0; i < lengthInBytes; i++, shift += 8) {
k1 ^= (base.getByte(i) & 0xFF) << shift;
@ -110,11 +116,10 @@ public final class Murmur3_x86_32 {
return fmix(h1, lengthInBytes);
}
private static int hashBytesByIntBlock(MemoryBlock base, int seed) {
long lengthInBytes = base.size();
private static int hashBytesByIntBlock(MemoryBlock base, int lengthInBytes, int seed) {
assert (lengthInBytes % 4 == 0);
int h1 = seed;
for (long i = 0; i < lengthInBytes; i += 4) {
for (int i = 0; i < lengthInBytes; i += 4) {
int halfWord = base.getInt(i);
int k1 = mixK1(halfWord);
h1 = mixH1(h1, k1);