Added unit tests for size estimation for specialized hash sets and maps.
This commit is contained in:
parent
62889c419c
commit
95c55df1c2
|
@ -2,8 +2,20 @@ package org.apache.spark.util.collection
|
||||||
|
|
||||||
import scala.collection.mutable.HashSet
|
import scala.collection.mutable.HashSet
|
||||||
import org.scalatest.FunSuite
|
import org.scalatest.FunSuite
|
||||||
|
import org.scalatest.matchers.ShouldMatchers
|
||||||
|
import org.apache.spark.util.SizeEstimator
|
||||||
|
|
||||||
class OpenHashMapSuite extends FunSuite {
|
class OpenHashMapSuite extends FunSuite with ShouldMatchers {
|
||||||
|
|
||||||
|
test("size for specialized, primitive value (int)") {
|
||||||
|
val capacity = 1024
|
||||||
|
val map = new OpenHashMap[String, Int](capacity)
|
||||||
|
val actualSize = SizeEstimator.estimate(map)
|
||||||
|
// 64 bit for pointers, 32 bit for ints, and 1 bit for the bitset.
|
||||||
|
val expectedSize = capacity * (64 + 32 + 1) / 8
|
||||||
|
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||||
|
actualSize should be <= (expectedSize * 1.1).toLong
|
||||||
|
}
|
||||||
|
|
||||||
test("initialization") {
|
test("initialization") {
|
||||||
val goodMap1 = new OpenHashMap[String, Int](1)
|
val goodMap1 = new OpenHashMap[String, Int](1)
|
||||||
|
|
|
@ -1,9 +1,27 @@
|
||||||
package org.apache.spark.util.collection
|
package org.apache.spark.util.collection
|
||||||
|
|
||||||
import org.scalatest.FunSuite
|
import org.scalatest.FunSuite
|
||||||
|
import org.scalatest.matchers.ShouldMatchers
|
||||||
|
|
||||||
|
import org.apache.spark.util.SizeEstimator
|
||||||
|
|
||||||
|
|
||||||
class OpenHashSetSuite extends FunSuite {
|
class OpenHashSetSuite extends FunSuite with ShouldMatchers {
|
||||||
|
|
||||||
|
test("size for specialized, primitive int") {
|
||||||
|
val loadFactor = 0.7
|
||||||
|
val set = new OpenHashSet[Int](64, loadFactor)
|
||||||
|
for (i <- 0 until 1024) {
|
||||||
|
set.add(i)
|
||||||
|
}
|
||||||
|
assert(set.size === 1024)
|
||||||
|
assert(set.capacity > 1024)
|
||||||
|
val actualSize = SizeEstimator.estimate(set)
|
||||||
|
// 32 bits for the ints + 1 bit for the bitset
|
||||||
|
val expectedSize = set.capacity * (32 + 1) / 8
|
||||||
|
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||||
|
actualSize should be <= (expectedSize * 1.1).toLong
|
||||||
|
}
|
||||||
|
|
||||||
test("primitive int") {
|
test("primitive int") {
|
||||||
val set = new OpenHashSet[Int]
|
val set = new OpenHashSet[Int]
|
||||||
|
|
|
@ -2,8 +2,20 @@ package org.apache.spark.util.collection
|
||||||
|
|
||||||
import scala.collection.mutable.HashSet
|
import scala.collection.mutable.HashSet
|
||||||
import org.scalatest.FunSuite
|
import org.scalatest.FunSuite
|
||||||
|
import org.scalatest.matchers.ShouldMatchers
|
||||||
|
import org.apache.spark.util.SizeEstimator
|
||||||
|
|
||||||
class PrimitiveKeyOpenHashSetSuite extends FunSuite {
|
class PrimitiveKeyOpenHashMapSuite extends FunSuite with ShouldMatchers {
|
||||||
|
|
||||||
|
test("size for specialized, primitive key, value (int, int)") {
|
||||||
|
val capacity = 1024
|
||||||
|
val map = new PrimitiveKeyOpenHashMap[Int, Int](capacity)
|
||||||
|
val actualSize = SizeEstimator.estimate(map)
|
||||||
|
// 32 bit for keys, 32 bit for values, and 1 bit for the bitset.
|
||||||
|
val expectedSize = capacity * (32 + 32 + 1) / 8
|
||||||
|
// Make sure we are not allocating a significant amount of memory beyond our expected.
|
||||||
|
actualSize should be <= (expectedSize * 1.1).toLong
|
||||||
|
}
|
||||||
|
|
||||||
test("initialization") {
|
test("initialization") {
|
||||||
val goodMap1 = new PrimitiveKeyOpenHashMap[Int, Int](1)
|
val goodMap1 = new PrimitiveKeyOpenHashMap[Int, Int](1)
|
Loading…
Reference in a new issue