ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in ColumnarArray.copy()

Browse source 
## What changes were proposed in this pull request?

In https://github.com/apache/spark/issues/23569, the copy method of `ColumnarArray` is implemented.
To further improve it, we can return `UnsafeArrayData` for `date`/`timestamp` type in `ColumnarArray.copy()`.

## How was this patch tested?

Unit test

Closes #23761 from gengliangwang/copyDateAndTS.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
This commit is contained in:
Gengliang Wang 2019-02-13 10:23:31 +09:00 committed by Takeshi Yamamuro
parent 5a7403623d
commit 72a349a95d
2 changed files with 30 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
View file

@ -56,9 +56,9 @@ public final class ColumnarArray extends ArrayData {
return UnsafeArrayData.fromPrimitiveArray(toByteArray());
} else if (dt instanceof ShortType) {
return UnsafeArrayData.fromPrimitiveArray(toShortArray());
} else if (dt instanceof IntegerType) {
} else if (dt instanceof IntegerType || dt instanceof DateType) {
return UnsafeArrayData.fromPrimitiveArray(toIntArray());
} else if (dt instanceof LongType) {
} else if (dt instanceof LongType || dt instanceof TimestampType) {
return UnsafeArrayData.fromPrimitiveArray(toLongArray());
} else if (dt instanceof FloatType) {
return UnsafeArrayData.fromPrimitiveArray(toFloatArray());

28
sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
View file

@ -108,6 +108,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}
testVectors("date", 10, DateType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendInt(i)
}
val array = new ColumnarArray(testVector, 0, 10)
val arrayCopy = array.copy()
(0 until 10).foreach { i =>
assert(array.get(i, DateType) === i)
assert(arrayCopy.get(i, DateType) === i)
}
}
testVectors("long", 10, LongType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendLong(i)
@ -122,6 +136,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}
testVectors("timestamp", 10, TimestampType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendLong(i)
}
val array = new ColumnarArray(testVector, 0, 10)
val arrayCopy = array.copy()
(0 until 10).foreach { i =>
assert(array.get(i, TimestampType) === i)
assert(arrayCopy.get(i, TimestampType) === i)
}
}
testVectors("float", 10, FloatType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendFloat(i.toFloat)