ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Changes to make size estimator more accurate. Fixes object size, pointer size

Browse source 
according to architecture and also aligns objects and arrays when computing
instance sizes. Verified using Eclipse Memory Analysis Tool (MAT)
This commit is contained in:
Shivaram Venkataraman 2012-08-11 02:16:15 -07:00
parent 1c5ae3edf2
commit 980585b220
1 changed files with 38 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
core/src/main/scala/spark/SizeEstimator.scala
View file

@ -19,8 +19,6 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet
* http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html * http://www.javaworld.com/javaworld/javaqa/2003-12/02-qa-1226-sizeof.html
*/ */
object SizeEstimator { object SizeEstimator {
private val OBJECT_SIZE = 8 // Minimum size of a java.lang.Object
private val POINTER_SIZE = 4 // Size of an object reference
// Sizes of primitive types // Sizes of primitive types
private val BYTE_SIZE = 1 private val BYTE_SIZE = 1
@ -32,6 +30,28 @@ object SizeEstimator {
private val FLOAT_SIZE = 4 private val FLOAT_SIZE = 4
private val DOUBLE_SIZE = 8 private val DOUBLE_SIZE = 8
// Object and pointer sizes are arch dependent
val is64bit = System.getProperty("os.arch").contains("64")
// Size of an object reference
// TODO: Get this from jvm/system property
val isCompressedOops = Runtime.getRuntime.maxMemory < (Integer.MAX_VALUE.toLong*2)
// Minimum size of a java.lang.Object
val OBJECT_SIZE = if (!is64bit) 8 else {
if(!isCompressedOops) {
16
} else {
12
}
}
val POINTER_SIZE = if (is64bit && !isCompressedOops) 8 else 4
// Alignment boundary for objects
// TODO: Is this arch dependent ?
private val ALIGN_SIZE = 8
// A cache of ClassInfo objects for each class // A cache of ClassInfo objects for each class
private val classInfos = new ConcurrentHashMap[Class[_], ClassInfo] private val classInfos = new ConcurrentHashMap[Class[_], ClassInfo]
classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil)) classInfos.put(classOf[Object], new ClassInfo(OBJECT_SIZE, Nil))
@ -101,10 +121,17 @@ object SizeEstimator {
private def visitArray(array: AnyRef, cls: Class[_], state: SearchState) { private def visitArray(array: AnyRef, cls: Class[_], state: SearchState) {
val length = JArray.getLength(array) val length = JArray.getLength(array)
val elementClass = cls.getComponentType val elementClass = cls.getComponentType
// Arrays have object header and length field which is an integer
var arrSize: Long = alignSize(OBJECT_SIZE + INT_SIZE)
if (elementClass.isPrimitive) { if (elementClass.isPrimitive) {
state.size += length * primitiveSize(elementClass) arrSize += alignSize(length * primitiveSize(elementClass))
state.size += arrSize
} else { } else {
state.size += length * POINTER_SIZE arrSize += alignSize(length * POINTER_SIZE)
state.size += arrSize
if (length <= ARRAY_SIZE_FOR_SAMPLING) { if (length <= ARRAY_SIZE_FOR_SAMPLING) {
for (i <- 0 until length) { for (i <- 0 until length) {
state.enqueue(JArray.get(array, i)) state.enqueue(JArray.get(array, i))
@ -176,9 +203,16 @@ object SizeEstimator {
} }
} }
shellSize = alignSize(shellSize)
// Create and cache a new ClassInfo // Create and cache a new ClassInfo
val newInfo = new ClassInfo(shellSize, pointerFields) val newInfo = new ClassInfo(shellSize, pointerFields)
classInfos.put(cls, newInfo) classInfos.put(cls, newInfo)
return newInfo return newInfo
} }
private def alignSize(size: Long): Long = {
val rem = size % ALIGN_SIZE
return if (rem == 0) size else (size + ALIGN_SIZE - rem)
}
} }