ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Revert "[SPARK-8928] [SQL] Makes CatalystSchemaConverter sticking to 1.4.x- when handling Parquet LISTs in compatible mode"

Browse source 
This reverts commit 3dab0da429.
This commit is contained in:
Cheng Lian 2015-07-08 22:14:38 -07:00
parent 3dab0da429
commit c056484c07
2 changed files with 7 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
View file

@ -461,8 +461,7 @@ private[parquet] class CatalystSchemaConverter(
field.name,
Types
.buildGroup(REPEATED)
// "array_element" is the name chosen by parquet-hive (1.7.0 and prior version)
.addField(convertField(StructField("array_element", elementType, nullable)))
.addField(convertField(StructField("element", elementType, nullable)))
.named(CatalystConverter.ARRAY_CONTAINS_NULL_BAG_SCHEMA_NAME))
// Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level
@ -475,8 +474,7 @@ private[parquet] class CatalystSchemaConverter(
ConversionPatterns.listType(
repetition,
field.name,
// "array" is the name chosen by parquet-avro (1.7.0 and prior version)
convertField(StructField("array", elementType, nullable), REPEATED))
convertField(StructField("element", elementType, nullable), REPEATED))
// Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by
// MAP_KEY_VALUE. This is covered by `convertGroupField(field: GroupType): DataType`.

10
sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
View file

@ -174,7 +174,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
"""
|message root {
| optional group _1 (LIST) {
| repeated int32 array;
| repeated int32 element;
| }
|}
""".stripMargin)
@ -198,7 +198,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
|message root {
| optional group _1 (LIST) {
| repeated group bag {
| optional int32 array_element;
| optional int32 element;
| }
| }
|}
@ -267,7 +267,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
| optional binary _1 (UTF8);
| optional group _2 (LIST) {
| repeated group bag {
| optional group array_element {
| optional group element {
| required int32 _1;
| required double _2;
| }
@ -616,7 +616,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
"""message root {
| optional group f1 (LIST) {
| repeated group bag {
| optional int32 array_element;
| optional int32 element;
| }
| }
|}
@ -648,7 +648,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
nullable = true))),
"""message root {
| optional group f1 (LIST) {
| repeated int32 array;
| repeated int32 element;
| }
|}
""".stripMargin)