[SPARK-11379][SQL] ExpressionEncoder can't handle top level primitive type correctly
For inner primitive type(e.g. inside `Product`), we use `schemaFor` to get the catalyst type for it, https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala#L403. However, for top level primitive type, we use `dataTypeFor`, which is wrong. Author: Wenchen Fan <wenchen@databricks.com> Closes #9337 from cloud-fan/encoder.
This commit is contained in:
parent
3dfa4ea526
commit
87f28fc240
|
@ -170,7 +170,7 @@ trait ScalaReflection {
|
||||||
.getOrElse(BoundReference(ordinal, dataType, false))
|
.getOrElse(BoundReference(ordinal, dataType, false))
|
||||||
|
|
||||||
/** Returns the current path or throws an error. */
|
/** Returns the current path or throws an error. */
|
||||||
def getPath = path.getOrElse(BoundReference(0, dataTypeFor(tpe), true))
|
def getPath = path.getOrElse(BoundReference(0, schemaFor(tpe).dataType, true))
|
||||||
|
|
||||||
tpe match {
|
tpe match {
|
||||||
case t if !dataTypeFor(t).isInstanceOf[ObjectType] =>
|
case t if !dataTypeFor(t).isInstanceOf[ObjectType] =>
|
||||||
|
|
|
@ -57,6 +57,7 @@ class ExpressionEncoderSuite extends SparkFunSuite {
|
||||||
encodeDecodeTest(false)
|
encodeDecodeTest(false)
|
||||||
encodeDecodeTest(1.toShort)
|
encodeDecodeTest(1.toShort)
|
||||||
encodeDecodeTest(1.toByte)
|
encodeDecodeTest(1.toByte)
|
||||||
|
encodeDecodeTest("hello")
|
||||||
|
|
||||||
encodeDecodeTest(PrimitiveData(1, 1, 1, 1, 1, 1, true))
|
encodeDecodeTest(PrimitiveData(1, 1, 1, 1, 1, 1, true))
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue