Added unit tests for size estimation for specialized hash sets and maps.
This commit is contained in:
parent
62889c419c
commit
95c55df1c2
|
@ -2,8 +2,20 @@ package org.apache.spark.util.collection
|
|||
|
||||
import scala.collection.mutable.HashSet
|
||||
import org.scalatest.FunSuite
|
||||
import org.scalatest.matchers.ShouldMatchers
|
||||
import org.apache.spark.util.SizeEstimator
|
||||
|
||||
class OpenHashMapSuite extends FunSuite {
|
||||
class OpenHashMapSuite extends FunSuite with ShouldMatchers {
|
||||
|
||||
test("size for specialized, primitive value (int)") {
|
||||
val capacity = 1024
|
||||
val map = new OpenHashMap[String, Int](capacity)
|
||||
val actualSize = SizeEstimator.estimate(map)
|
||||
// 64 bit for pointers, 32 bit for ints, and 1 bit for the bitset.
|
||||
val expectedSize = capacity * (64 + 32 + 1) / 8
|
||||
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||
actualSize should be <= (expectedSize * 1.1).toLong
|
||||
}
|
||||
|
||||
test("initialization") {
|
||||
val goodMap1 = new OpenHashMap[String, Int](1)
|
||||
|
|
|
@ -1,9 +1,27 @@
|
|||
package org.apache.spark.util.collection
|
||||
|
||||
import org.scalatest.FunSuite
|
||||
import org.scalatest.matchers.ShouldMatchers
|
||||
|
||||
import org.apache.spark.util.SizeEstimator
|
||||
|
||||
|
||||
class OpenHashSetSuite extends FunSuite {
|
||||
class OpenHashSetSuite extends FunSuite with ShouldMatchers {
|
||||
|
||||
test("size for specialized, primitive int") {
|
||||
val loadFactor = 0.7
|
||||
val set = new OpenHashSet[Int](64, loadFactor)
|
||||
for (i <- 0 until 1024) {
|
||||
set.add(i)
|
||||
}
|
||||
assert(set.size === 1024)
|
||||
assert(set.capacity > 1024)
|
||||
val actualSize = SizeEstimator.estimate(set)
|
||||
// 32 bits for the ints + 1 bit for the bitset
|
||||
val expectedSize = set.capacity * (32 + 1) / 8
|
||||
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||
actualSize should be <= (expectedSize * 1.1).toLong
|
||||
}
|
||||
|
||||
test("primitive int") {
|
||||
val set = new OpenHashSet[Int]
|
||||
|
|
|
@ -2,8 +2,20 @@ package org.apache.spark.util.collection
|
|||
|
||||
import scala.collection.mutable.HashSet
|
||||
import org.scalatest.FunSuite
|
||||
import org.scalatest.matchers.ShouldMatchers
|
||||
import org.apache.spark.util.SizeEstimator
|
||||
|
||||
class PrimitiveKeyOpenHashSetSuite extends FunSuite {
|
||||
class PrimitiveKeyOpenHashMapSuite extends FunSuite with ShouldMatchers {
|
||||
|
||||
test("size for specialized, primitive key, value (int, int)") {
|
||||
val capacity = 1024
|
||||
val map = new PrimitiveKeyOpenHashMap[Int, Int](capacity)
|
||||
val actualSize = SizeEstimator.estimate(map)
|
||||
// 32 bit for keys, 32 bit for values, and 1 bit for the bitset.
|
||||
val expectedSize = capacity * (32 + 32 + 1) / 8
|
||||
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||
actualSize should be <= (expectedSize * 1.1).toLong
|
||||
}
|
||||
|
||||
test("initialization") {
|
||||
val goodMap1 = new PrimitiveKeyOpenHashMap[Int, Int](1)
|
Loading…
Reference in a new issue