[SPARK-9831] [SQL] fix serialization with empty broadcast

Author: Davies Liu <davies@databricks.com>

Closes #8117 from davies/fix_serialization and squashes the following commits:

d21ac71 [Davies Liu] fix serialization with empty broadcast
This commit is contained in:
Davies Liu 2015-08-11 22:45:18 -07:00 committed by Davies Liu
parent 74a293f453
commit c3e9a120e3
2 changed files with 18 additions and 1 deletions

View file

@ -299,7 +299,7 @@ private[joins] final class UnsafeHashedRelation(
binaryMap = new BytesToBytesMap(
taskMemoryManager,
shuffleMemoryManager,
nKeys * 2, // reduce hash collision
(nKeys * 1.5 + 1).toInt, // reduce hash collision
pageSizeBytes)
var i = 0

View file

@ -103,4 +103,21 @@ class HashedRelationSuite extends SparkFunSuite {
assert(hashed2.get(unsafeData(2)) === data2)
assert(numDataRows.value.value === data.length)
}
test("test serialization empty hash map") {
val os = new ByteArrayOutputStream()
val out = new ObjectOutputStream(os)
val hashed = new UnsafeHashedRelation(
new java.util.HashMap[UnsafeRow, CompactBuffer[UnsafeRow]])
hashed.writeExternal(out)
out.flush()
val in = new ObjectInputStream(new ByteArrayInputStream(os.toByteArray))
val hashed2 = new UnsafeHashedRelation()
hashed2.readExternal(in)
val schema = StructType(StructField("a", IntegerType, true) :: Nil)
val toUnsafe = UnsafeProjection.create(schema)
val row = toUnsafe(InternalRow(0))
assert(hashed2.get(row) === null)
}
}