From 980585b220013f2f4effcf0242b22f6c82674aa9 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 11 Aug 2012 02:16:15 -0700 Subject: [PATCH 1/7] Changes to make size estimator more accurate. Fixes object size, pointer size according to architecture and also aligns objects and arrays when computing instance sizes. Verified using Eclipse Memory Analysis Tool (MAT) --- core/src/main/scala/spark/SizeEstimator.scala | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index b3bd4daa73..f196a6d818 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -19,8 +19,6 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet * http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html */ object SizeEstimator { - private val OBJECT_SIZE = 8 // Minimum size of a java.lang.Object - private val POINTER_SIZE = 4 // Size of an object reference // Sizes of primitive types private val BYTE_SIZE = 1 @@ -32,6 +30,28 @@ object SizeEstimator { private val FLOAT_SIZE = 4 private val DOUBLE_SIZE = 8 + // Object and pointer sizes are arch dependent + val is64bit = System.getProperty("os.arch").contains("64") + + // Size of an object reference + // TODO: Get this from jvm/system property + val isCompressedOops = Runtime.getRuntime.maxMemory < (Integer.MAX_VALUE.toLong*2) + + // Minimum size of a java.lang.Object + val OBJECT_SIZE = if (!is64bit) 8 else { + if(!isCompressedOops) { + 16 + } else { + 12 + } + } + + val POINTER_SIZE = if (is64bit && !isCompressedOops) 8 else 4 + + // Alignment boundary for objects + // TODO: Is this arch dependent ? + private val ALIGN_SIZE = 8 + // A cache of ClassInfo objects for each class private val classInfos = new ConcurrentHashMap[Class[_], ClassInfo] classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil)) @@ -101,10 +121,17 @@ object SizeEstimator { private def visitArray(array: AnyRef, cls: Class[_], state: SearchState) { val length = JArray.getLength(array) val elementClass = cls.getComponentType + + // Arrays have object header and length field which is an integer + var arrSize: Long = alignSize(OBJECT_SIZE + INT_SIZE) + if (elementClass.isPrimitive) { - state.size += length * primitiveSize(elementClass) + arrSize += alignSize(length * primitiveSize(elementClass)) + state.size += arrSize } else { - state.size += length * POINTER_SIZE + arrSize += alignSize(length * POINTER_SIZE) + state.size += arrSize + if (length <= ARRAY_SIZE_FOR_SAMPLING) { for (i <- 0 until length) { state.enqueue(JArray.get(array, i)) @@ -176,9 +203,16 @@ object SizeEstimator { } } + shellSize = alignSize(shellSize) + // Create and cache a new ClassInfo val newInfo = new ClassInfo(shellSize, pointerFields) classInfos.put(cls, newInfo) return newInfo } + + private def alignSize(size: Long): Long = { + val rem = size % ALIGN_SIZE + return if (rem == 0) size else (size + ALIGN_SIZE - rem) + } } From f2475ca95a6f0e5035fe2a5b29989f5783c654e9 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 11 Aug 2012 02:34:20 -0700 Subject: [PATCH 2/7] Add link to Java wiki which specifies what changes with compressed oops --- core/src/main/scala/spark/SizeEstimator.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index f196a6d818..3a63b236f3 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -37,6 +37,9 @@ object SizeEstimator { // TODO: Get this from jvm/system property val isCompressedOops = Runtime.getRuntime.maxMemory < (Integer.MAX_VALUE.toLong*2) + // Based on https://wikis.oracle.com/display/HotSpotInternals/CompressedOops + // section, "Which oops are compressed" + // Minimum size of a java.lang.Object val OBJECT_SIZE = if (!is64bit) 8 else { if(!isCompressedOops) { From c0e773aa01408ce06c509f9ab9fc736e5a3d3071 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 11 Aug 2012 14:38:05 -0700 Subject: [PATCH 3/7] Use HotSpotDiagnosticMXBean to get if CompressedOops are in use or not --- core/src/main/scala/spark/SizeEstimator.scala | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index 3a63b236f3..d43558dc09 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -7,6 +7,10 @@ import java.util.IdentityHashMap import java.util.concurrent.ConcurrentHashMap import java.util.Random +import javax.management.MBeanServer +import java.lang.management.ManagementFactory +import com.sun.management.HotSpotDiagnosticMXBean + import scala.collection.mutable.ArrayBuffer import it.unimi.dsi.fastutil.ints.IntOpenHashSet @@ -18,7 +22,7 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet * Based on the following JavaWorld article: * http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html */ -object SizeEstimator { +object SizeEstimator extends Logging { // Sizes of primitive types private val BYTE_SIZE = 1 @@ -34,8 +38,7 @@ object SizeEstimator { val is64bit = System.getProperty("os.arch").contains("64") // Size of an object reference - // TODO: Get this from jvm/system property - val isCompressedOops = Runtime.getRuntime.maxMemory < (Integer.MAX_VALUE.toLong*2) + val isCompressedOops = getIsCompressedOops // Based on https://wikis.oracle.com/display/HotSpotInternals/CompressedOops // section, "Which oops are compressed" @@ -59,6 +62,28 @@ object SizeEstimator { private val classInfos = new ConcurrentHashMap[Class[_], ClassInfo] classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil)) + private def getIsCompressedOops : Boolean = { + try { + val hotSpotMBeanName = "com.sun.management:type=HotSpotDiagnostic"; + val server = ManagementFactory.getPlatformMBeanServer(); + val bean = ManagementFactory.newPlatformMXBeanProxy(server, + hotSpotMBeanName, classOf[HotSpotDiagnosticMXBean]); + return bean.getVMOption("UseCompressedOops").getValue.toBoolean + } catch { + case e: IllegalArgumentException => { + logWarning("Exception while trying to check if compressed oops is enabled", e) + // Fall back to checking if maxMemory < 32GB + return Runtime.getRuntime.maxMemory < (32L*1024*1024*1024) + } + + case e: SecurityException => { + logWarning("No permission to create MBeanServer", e) + // Fall back to checking if maxMemory < 32GB + return Runtime.getRuntime.maxMemory < (32L*1024*1024*1024) + } + } + } + /** * The state of an ongoing size estimation. Contains a stack of objects to visit as well as an * IdentityHashMap of visited objects, and provides utility methods for enqueueing new objects From 64b8fd62f0de1f789c5a48ef8b60fdbb2c875704 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 11 Aug 2012 16:40:33 -0700 Subject: [PATCH 4/7] If spark.test.useCompressedOops is set, use that to infer compressed oops setting. This is useful to get a deterministic test case --- core/src/main/scala/spark/SizeEstimator.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index d43558dc09..45f9a1cd40 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -63,6 +63,9 @@ object SizeEstimator extends Logging { classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil)) private def getIsCompressedOops : Boolean = { + if (System.getProperty("spark.test.useCompressedOops") != null) { + return System.getProperty("spark.test.useCompressedOops").toBoolean + } try { val hotSpotMBeanName = "com.sun.management:type=HotSpotDiagnostic"; val server = ManagementFactory.getPlatformMBeanServer(); From 73452cc64989752c4aa8e3a05abed314a5b6b985 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 11 Aug 2012 16:42:35 -0700 Subject: [PATCH 5/7] Update test cases to match the new size estimates. Uses 64-bit and compressed oops setting to get deterministic results --- .../scala/spark/BoundedMemoryCacheSuite.scala | 26 ++++-- .../test/scala/spark/SizeEstimatorSuite.scala | 86 ++++++++++++------- 2 files changed, 77 insertions(+), 35 deletions(-) diff --git a/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala b/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala index 024ce0b8d1..745c86a0d0 100644 --- a/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala +++ b/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala @@ -4,28 +4,44 @@ import org.scalatest.FunSuite class BoundedMemoryCacheSuite extends FunSuite { test("constructor test") { - val cache = new BoundedMemoryCache(40) - expect(40)(cache.getCapacity) + val cache = new BoundedMemoryCache(60) + expect(60)(cache.getCapacity) } test("caching") { - val cache = new BoundedMemoryCache(40) { + // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case + val oldArch = System.setProperty("os.arch", "amd64") + val oldOops = System.setProperty("spark.test.useCompressedOops", "true") + + val cache = new BoundedMemoryCache(60) { //TODO sorry about this, but there is not better way how to skip 'cacheTracker.dropEntry' override protected def reportEntryDropped(datasetId: Any, partition: Int, entry: Entry) { logInfo("Dropping key (%s, %d) of size %d to make space".format(datasetId, partition, entry.size)) } } //should be OK - expect(CachePutSuccess(30))(cache.put("1", 0, "Meh")) + expect(CachePutSuccess(56))(cache.put("1", 0, "Meh")) //we cannot add this to cache (there is not enough space in cache) & we cannot evict the only value from //cache because it's from the same dataset expect(CachePutFailure())(cache.put("1", 1, "Meh")) //should be OK, dataset '1' can be evicted from cache - expect(CachePutSuccess(30))(cache.put("2", 0, "Meh")) + expect(CachePutSuccess(56))(cache.put("2", 0, "Meh")) //should fail, cache should obey it's capacity expect(CachePutFailure())(cache.put("3", 0, "Very_long_and_useless_string")) + + if (oldArch != null) { + System.setProperty("os.arch", oldArch) + } else { + System.clearProperty("os.arch") + } + + if (oldOops != null) { + System.setProperty("spark.test.useCompressedOops", oldOops) + } else { + System.clearProperty("spark.test.useCompressedOops") + } } } diff --git a/core/src/test/scala/spark/SizeEstimatorSuite.scala b/core/src/test/scala/spark/SizeEstimatorSuite.scala index 63bc951858..9c45b3c287 100644 --- a/core/src/test/scala/spark/SizeEstimatorSuite.scala +++ b/core/src/test/scala/spark/SizeEstimatorSuite.scala @@ -1,6 +1,7 @@ package spark import org.scalatest.FunSuite +import org.scalatest.BeforeAndAfterAll class DummyClass1 {} @@ -17,61 +18,86 @@ class DummyClass4(val d: DummyClass3) { val x: Int = 0 } -class SizeEstimatorSuite extends FunSuite { +class SizeEstimatorSuite extends FunSuite with BeforeAndAfterAll { + var oldArch: String = _ + var oldOops: String = _ + + override def beforeAll() { + // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case + oldArch = System.setProperty("os.arch", "amd64") + oldOops = System.setProperty("spark.test.useCompressedOops", "true") + } + + override def afterAll() { + if (oldArch != null) { + System.setProperty("os.arch", oldArch) + } else { + System.clearProperty("os.arch") + } + + if (oldOops != null) { + System.setProperty("spark.test.useCompressedOops", oldOops) + } else { + System.clearProperty("spark.test.useCompressedOops") + } + } + test("simple classes") { - expect(8)(SizeEstimator.estimate(new DummyClass1)) - expect(12)(SizeEstimator.estimate(new DummyClass2)) - expect(20)(SizeEstimator.estimate(new DummyClass3)) - expect(16)(SizeEstimator.estimate(new DummyClass4(null))) - expect(36)(SizeEstimator.estimate(new DummyClass4(new DummyClass3))) + expect(16)(SizeEstimator.estimate(new DummyClass1)) + expect(16)(SizeEstimator.estimate(new DummyClass2)) + expect(24)(SizeEstimator.estimate(new DummyClass3)) + expect(24)(SizeEstimator.estimate(new DummyClass4(null))) + expect(48)(SizeEstimator.estimate(new DummyClass4(new DummyClass3))) } test("strings") { - expect(24)(SizeEstimator.estimate("")) - expect(26)(SizeEstimator.estimate("a")) - expect(28)(SizeEstimator.estimate("ab")) - expect(40)(SizeEstimator.estimate("abcdefgh")) + expect(48)(SizeEstimator.estimate("")) + expect(56)(SizeEstimator.estimate("a")) + expect(56)(SizeEstimator.estimate("ab")) + expect(64)(SizeEstimator.estimate("abcdefgh")) } test("primitive arrays") { - expect(10)(SizeEstimator.estimate(new Array[Byte](10))) - expect(20)(SizeEstimator.estimate(new Array[Char](10))) - expect(20)(SizeEstimator.estimate(new Array[Short](10))) - expect(40)(SizeEstimator.estimate(new Array[Int](10))) - expect(80)(SizeEstimator.estimate(new Array[Long](10))) - expect(40)(SizeEstimator.estimate(new Array[Float](10))) - expect(80)(SizeEstimator.estimate(new Array[Double](10))) - expect(4000)(SizeEstimator.estimate(new Array[Int](1000))) - expect(8000)(SizeEstimator.estimate(new Array[Long](1000))) + expect(32)(SizeEstimator.estimate(new Array[Byte](10))) + expect(40)(SizeEstimator.estimate(new Array[Char](10))) + expect(40)(SizeEstimator.estimate(new Array[Short](10))) + expect(56)(SizeEstimator.estimate(new Array[Int](10))) + expect(96)(SizeEstimator.estimate(new Array[Long](10))) + expect(56)(SizeEstimator.estimate(new Array[Float](10))) + expect(96)(SizeEstimator.estimate(new Array[Double](10))) + expect(4016)(SizeEstimator.estimate(new Array[Int](1000))) + expect(8016)(SizeEstimator.estimate(new Array[Long](1000))) } test("object arrays") { // Arrays containing nulls should just have one pointer per element - expect(40)(SizeEstimator.estimate(new Array[String](10))) - expect(40)(SizeEstimator.estimate(new Array[AnyRef](10))) + expect(56)(SizeEstimator.estimate(new Array[String](10))) + expect(56)(SizeEstimator.estimate(new Array[AnyRef](10))) // For object arrays with non-null elements, each object should take one pointer plus // however many bytes that class takes. (Note that Array.fill calls the code in its // second parameter separately for each object, so we get distinct objects.) - expect(120)(SizeEstimator.estimate(Array.fill(10)(new DummyClass1))) - expect(160)(SizeEstimator.estimate(Array.fill(10)(new DummyClass2))) - expect(240)(SizeEstimator.estimate(Array.fill(10)(new DummyClass3))) - expect(12 + 16)(SizeEstimator.estimate(Array(new DummyClass1, new DummyClass2))) + expect(216)(SizeEstimator.estimate(Array.fill(10)(new DummyClass1))) + expect(216)(SizeEstimator.estimate(Array.fill(10)(new DummyClass2))) + expect(296)(SizeEstimator.estimate(Array.fill(10)(new DummyClass3))) + expect(56)(SizeEstimator.estimate(Array(new DummyClass1, new DummyClass2))) // Past size 100, our samples 100 elements, but we should still get the right size. - expect(24000)(SizeEstimator.estimate(Array.fill(1000)(new DummyClass3))) + expect(28016)(SizeEstimator.estimate(Array.fill(1000)(new DummyClass3))) // If an array contains the *same* element many times, we should only count it once. val d1 = new DummyClass1 - expect(48)(SizeEstimator.estimate(Array.fill(10)(d1))) // 10 pointers plus 8-byte object - expect(408)(SizeEstimator.estimate(Array.fill(100)(d1))) // 100 pointers plus 8-byte object + expect(72)(SizeEstimator.estimate(Array.fill(10)(d1))) // 10 pointers plus 8-byte object + expect(432)(SizeEstimator.estimate(Array.fill(100)(d1))) // 100 pointers plus 8-byte object // Same thing with huge array containing the same element many times. Note that this won't - // return exactly 4008 because it can't tell that *all* the elements will equal the first + // return exactly 4032 because it can't tell that *all* the elements will equal the first // one it samples, but it should be close to that. + + // TODO: If we sample 100 elements, this should always be 4176 ? val estimatedSize = SizeEstimator.estimate(Array.fill(1000)(d1)) assert(estimatedSize >= 4000, "Estimated size " + estimatedSize + " should be more than 4000") - assert(estimatedSize <= 4100, "Estimated size " + estimatedSize + " should be less than 4100") + assert(estimatedSize <= 4200, "Estimated size " + estimatedSize + " should be less than 4100") } } From 54502238a22309686f3b48c7ea8e23be0d56de9c Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sun, 12 Aug 2012 17:16:27 -0700 Subject: [PATCH 6/7] Move object size and pointer size initialization into a function to enable unit-testing --- core/src/main/scala/spark/SizeEstimator.scala | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/core/src/main/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala index 45f9a1cd40..e5ad8b52dc 100644 --- a/core/src/main/scala/spark/SizeEstimator.scala +++ b/core/src/main/scala/spark/SizeEstimator.scala @@ -34,33 +34,43 @@ object SizeEstimator extends Logging { private val FLOAT_SIZE = 4 private val DOUBLE_SIZE = 8 - // Object and pointer sizes are arch dependent - val is64bit = System.getProperty("os.arch").contains("64") - - // Size of an object reference - val isCompressedOops = getIsCompressedOops - - // Based on https://wikis.oracle.com/display/HotSpotInternals/CompressedOops - // section, "Which oops are compressed" - - // Minimum size of a java.lang.Object - val OBJECT_SIZE = if (!is64bit) 8 else { - if(!isCompressedOops) { - 16 - } else { - 12 - } - } - - val POINTER_SIZE = if (is64bit && !isCompressedOops) 8 else 4 - // Alignment boundary for objects // TODO: Is this arch dependent ? private val ALIGN_SIZE = 8 // A cache of ClassInfo objects for each class private val classInfos = new ConcurrentHashMap[Class[_], ClassInfo] - classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil)) + + // Object and pointer sizes are arch dependent + private var is64bit = false + + // Size of an object reference + // Based on https://wikis.oracle.com/display/HotSpotInternals/CompressedOops + private var isCompressedOops = false + private var pointerSize = 4 + + // Minimum size of a java.lang.Object + private var objectSize = 8 + + initialize() + + // Sets object size, pointer size based on architecture and CompressedOops settings + // from the JVM. + private def initialize() { + is64bit = System.getProperty("os.arch").contains("64") + isCompressedOops = getIsCompressedOops + + objectSize = if (!is64bit) 8 else { + if(!isCompressedOops) { + 16 + } else { + 12 + } + } + pointerSize = if (is64bit && !isCompressedOops) 8 else 4 + classInfos.clear() + classInfos.put(classOf[Object], new ClassInfo(objectSize, Nil)) + } private def getIsCompressedOops : Boolean = { if (System.getProperty("spark.test.useCompressedOops") != null) { @@ -154,13 +164,13 @@ object SizeEstimator extends Logging { val elementClass = cls.getComponentType // Arrays have object header and length field which is an integer - var arrSize: Long = alignSize(OBJECT_SIZE + INT_SIZE) + var arrSize: Long = alignSize(objectSize + INT_SIZE) if (elementClass.isPrimitive) { arrSize += alignSize(length * primitiveSize(elementClass)) state.size += arrSize } else { - arrSize += alignSize(length * POINTER_SIZE) + arrSize += alignSize(length * pointerSize) state.size += arrSize if (length <= ARRAY_SIZE_FOR_SAMPLING) { @@ -228,7 +238,7 @@ object SizeEstimator extends Logging { shellSize += primitiveSize(fieldClass) } else { field.setAccessible(true) // Enable future get()'s on this field - shellSize += POINTER_SIZE + shellSize += pointerSize pointerFields = field :: pointerFields } } From 2ee731211a67576549b970639629c02bf8dad338 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sun, 12 Aug 2012 17:18:01 -0700 Subject: [PATCH 7/7] Add test-cases for 32-bit and no-compressed oops scenarios. --- .../scala/spark/BoundedMemoryCacheSuite.scala | 5 +- .../test/scala/spark/SizeEstimatorSuite.scala | 55 ++++++++++++++----- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala b/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala index 745c86a0d0..dff2970566 100644 --- a/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala +++ b/core/src/test/scala/spark/BoundedMemoryCacheSuite.scala @@ -1,8 +1,9 @@ package spark import org.scalatest.FunSuite +import org.scalatest.PrivateMethodTester -class BoundedMemoryCacheSuite extends FunSuite { +class BoundedMemoryCacheSuite extends FunSuite with PrivateMethodTester { test("constructor test") { val cache = new BoundedMemoryCache(60) expect(60)(cache.getCapacity) @@ -12,6 +13,8 @@ class BoundedMemoryCacheSuite extends FunSuite { // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case val oldArch = System.setProperty("os.arch", "amd64") val oldOops = System.setProperty("spark.test.useCompressedOops", "true") + val initialize = PrivateMethod[Unit]('initialize) + SizeEstimator invokePrivate initialize() val cache = new BoundedMemoryCache(60) { //TODO sorry about this, but there is not better way how to skip 'cacheTracker.dropEntry' diff --git a/core/src/test/scala/spark/SizeEstimatorSuite.scala b/core/src/test/scala/spark/SizeEstimatorSuite.scala index 9c45b3c287..a2015644ee 100644 --- a/core/src/test/scala/spark/SizeEstimatorSuite.scala +++ b/core/src/test/scala/spark/SizeEstimatorSuite.scala @@ -2,6 +2,7 @@ package spark import org.scalatest.FunSuite import org.scalatest.BeforeAndAfterAll +import org.scalatest.PrivateMethodTester class DummyClass1 {} @@ -18,7 +19,7 @@ class DummyClass4(val d: DummyClass3) { val x: Int = 0 } -class SizeEstimatorSuite extends FunSuite with BeforeAndAfterAll { +class SizeEstimatorSuite extends FunSuite with BeforeAndAfterAll with PrivateMethodTester { var oldArch: String = _ var oldOops: String = _ @@ -29,17 +30,8 @@ class SizeEstimatorSuite extends FunSuite with BeforeAndAfterAll { } override def afterAll() { - if (oldArch != null) { - System.setProperty("os.arch", oldArch) - } else { - System.clearProperty("os.arch") - } - - if (oldOops != null) { - System.setProperty("spark.test.useCompressedOops", oldOops) - } else { - System.clearProperty("spark.test.useCompressedOops") - } + resetOrClear("os.arch", oldArch) + resetOrClear("spark.test.useCompressedOops", oldOops) } test("simple classes") { @@ -99,5 +91,42 @@ class SizeEstimatorSuite extends FunSuite with BeforeAndAfterAll { assert(estimatedSize >= 4000, "Estimated size " + estimatedSize + " should be more than 4000") assert(estimatedSize <= 4200, "Estimated size " + estimatedSize + " should be less than 4100") } -} + test("32-bit arch") { + val arch = System.setProperty("os.arch", "x86") + + val initialize = PrivateMethod[Unit]('initialize) + SizeEstimator invokePrivate initialize() + + expect(40)(SizeEstimator.estimate("")) + expect(48)(SizeEstimator.estimate("a")) + expect(48)(SizeEstimator.estimate("ab")) + expect(56)(SizeEstimator.estimate("abcdefgh")) + + resetOrClear("os.arch", arch) + } + + test("64-bit arch with no compressed oops") { + val arch = System.setProperty("os.arch", "amd64") + val oops = System.setProperty("spark.test.useCompressedOops", "false") + + val initialize = PrivateMethod[Unit]('initialize) + SizeEstimator invokePrivate initialize() + + expect(64)(SizeEstimator.estimate("")) + expect(72)(SizeEstimator.estimate("a")) + expect(72)(SizeEstimator.estimate("ab")) + expect(80)(SizeEstimator.estimate("abcdefgh")) + + resetOrClear("os.arch", arch) + resetOrClear("spark.test.useCompressedOops", oops) + } + + def resetOrClear(prop: String, oldValue: String) { + if (oldValue != null) { + System.setProperty(prop, oldValue) + } else { + System.clearProperty(prop) + } + } +}