ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[HOTFIX] Fix Java 7 compilation break

Browse source
This commit is contained in:
Reynold Xin 2016-04-21 17:52:10 -07:00
parent f181aee07c
commit 0bf8df250e
4 changed files with 6 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
core/src/main/java/org/apache/spark/shuffle/sort/ShuffleInMemorySorter.java
View file

@ -17,7 +17,6 @@
package org.apache.spark.shuffle.sort;
import java.lang.Long;
import java.util.Comparator;
import org.apache.spark.memory.MemoryConsumer;
@ -102,7 +101,7 @@ final class ShuffleInMemorySorter {
array.getBaseOffset(),
newArray.getBaseObject(),
newArray.getBaseOffset(),
array.size() * (Long.BYTES / memoryAllocationFactor)
array.size() * (8 / memoryAllocationFactor)
);
consumer.freeArray(array);
array = newArray;
@ -113,7 +112,7 @@ final class ShuffleInMemorySorter {
}
public long getMemoryUsage() {
return array.size() * Long.BYTES;
return array.size() * 8;
}
/**

6
core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
View file

@ -17,7 +17,6 @@
package org.apache.spark.util.collection.unsafe.sort;
import java.lang.Long;
import java.util.Comparator;
import org.apache.avro.reflect.Nullable;
@ -27,7 +26,6 @@ import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.LongArray;
import org.apache.spark.util.collection.Sorter;
import org.apache.spark.util.collection.unsafe.sort.RadixSort;
/**
* Sorts records using an AlphaSort-style key-prefix sort. This sort stores pointers to records
@ -163,7 +161,7 @@ public final class UnsafeInMemorySorter {
}
public long getMemoryUsage() {
return array.size() * Long.BYTES;
return array.size() * 8;
}
public boolean hasSpaceForAnotherRecord() {
@ -179,7 +177,7 @@ public final class UnsafeInMemorySorter {
array.getBaseOffset(),
newArray.getBaseObject(),
newArray.getBaseOffset(),
array.size() * (Long.BYTES / memoryAllocationFactor));
array.size() * (8 / memoryAllocationFactor));
consumer.freeArray(array);
array = newArray;
}

3
core/src/test/java/org/apache/spark/shuffle/sort/ShuffleInMemorySorterSuite.java
View file

@ -17,7 +17,6 @@
package org.apache.spark.shuffle.sort;
import java.lang.Long;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Random;
@ -83,7 +82,7 @@ public class ShuffleInMemorySorterSuite {
for (String str : dataToSort) {
if (!sorter.hasSpaceForAnotherRecord()) {
sorter.expandPointerArray(
consumer.allocateArray(sorter.getMemoryUsage() / Long.BYTES * 2));
consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2));
}
final long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, position);
final byte[] strBytes = str.getBytes(StandardCharsets.UTF_8);

3
core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
View file

@ -17,7 +17,6 @@
package org.apache.spark.util.collection.unsafe.sort;
import java.lang.Long;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@ -114,7 +113,7 @@ public class UnsafeInMemorySorterSuite {
for (int i = 0; i < dataToSort.length; i++) {
if (!sorter.hasSpaceForAnotherRecord()) {
sorter.expandPointerArray(
consumer.allocateArray(sorter.getMemoryUsage() / Long.BYTES * 2));
consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2));
}
// position now points to the start of a record (which holds its length).
final int recordLength = Platform.getInt(baseObject, position);