[SPARK-9831] [SQL] fix serialization with empty broadcast
Author: Davies Liu <davies@databricks.com> Closes #8117 from davies/fix_serialization and squashes the following commits: d21ac71 [Davies Liu] fix serialization with empty broadcast
This commit is contained in:
parent
74a293f453
commit
c3e9a120e3
|
@ -299,7 +299,7 @@ private[joins] final class UnsafeHashedRelation(
|
|||
binaryMap = new BytesToBytesMap(
|
||||
taskMemoryManager,
|
||||
shuffleMemoryManager,
|
||||
nKeys * 2, // reduce hash collision
|
||||
(nKeys * 1.5 + 1).toInt, // reduce hash collision
|
||||
pageSizeBytes)
|
||||
|
||||
var i = 0
|
||||
|
|
|
@ -103,4 +103,21 @@ class HashedRelationSuite extends SparkFunSuite {
|
|||
assert(hashed2.get(unsafeData(2)) === data2)
|
||||
assert(numDataRows.value.value === data.length)
|
||||
}
|
||||
|
||||
test("test serialization empty hash map") {
|
||||
val os = new ByteArrayOutputStream()
|
||||
val out = new ObjectOutputStream(os)
|
||||
val hashed = new UnsafeHashedRelation(
|
||||
new java.util.HashMap[UnsafeRow, CompactBuffer[UnsafeRow]])
|
||||
hashed.writeExternal(out)
|
||||
out.flush()
|
||||
val in = new ObjectInputStream(new ByteArrayInputStream(os.toByteArray))
|
||||
val hashed2 = new UnsafeHashedRelation()
|
||||
hashed2.readExternal(in)
|
||||
|
||||
val schema = StructType(StructField("a", IntegerType, true) :: Nil)
|
||||
val toUnsafe = UnsafeProjection.create(schema)
|
||||
val row = toUnsafe(InternalRow(0))
|
||||
assert(hashed2.get(row) === null)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue