85f9a61357
I caught SPARK-10136 while adding more test cases to `ParquetAvroCompatibilitySuite`. Actual bug fix code lies in `CatalystRowConverter.scala`. Author: Cheng Lian <lian@databricks.com> Closes #8341 from liancheng/spark-10136/parquet-avro-nested-primitive-array.
147 lines
3.1 KiB
Plaintext
147 lines
3.1 KiB
Plaintext
{
|
|
"protocol" : "CompatibilityTest",
|
|
"namespace" : "org.apache.spark.sql.execution.datasources.parquet.test.avro",
|
|
"types" : [ {
|
|
"type" : "enum",
|
|
"name" : "Suit",
|
|
"symbols" : [ "SPADES", "HEARTS", "DIAMONDS", "CLUBS" ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "ParquetEnum",
|
|
"fields" : [ {
|
|
"name" : "suit",
|
|
"type" : "Suit"
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "Nested",
|
|
"fields" : [ {
|
|
"name" : "nested_ints_column",
|
|
"type" : {
|
|
"type" : "array",
|
|
"items" : "int"
|
|
}
|
|
}, {
|
|
"name" : "nested_string_column",
|
|
"type" : "string"
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "AvroPrimitives",
|
|
"fields" : [ {
|
|
"name" : "bool_column",
|
|
"type" : "boolean"
|
|
}, {
|
|
"name" : "int_column",
|
|
"type" : "int"
|
|
}, {
|
|
"name" : "long_column",
|
|
"type" : "long"
|
|
}, {
|
|
"name" : "float_column",
|
|
"type" : "float"
|
|
}, {
|
|
"name" : "double_column",
|
|
"type" : "double"
|
|
}, {
|
|
"name" : "binary_column",
|
|
"type" : "bytes"
|
|
}, {
|
|
"name" : "string_column",
|
|
"type" : "string"
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "AvroOptionalPrimitives",
|
|
"fields" : [ {
|
|
"name" : "maybe_bool_column",
|
|
"type" : [ "null", "boolean" ]
|
|
}, {
|
|
"name" : "maybe_int_column",
|
|
"type" : [ "null", "int" ]
|
|
}, {
|
|
"name" : "maybe_long_column",
|
|
"type" : [ "null", "long" ]
|
|
}, {
|
|
"name" : "maybe_float_column",
|
|
"type" : [ "null", "float" ]
|
|
}, {
|
|
"name" : "maybe_double_column",
|
|
"type" : [ "null", "double" ]
|
|
}, {
|
|
"name" : "maybe_binary_column",
|
|
"type" : [ "null", "bytes" ]
|
|
}, {
|
|
"name" : "maybe_string_column",
|
|
"type" : [ "null", "string" ]
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "AvroNonNullableArrays",
|
|
"fields" : [ {
|
|
"name" : "strings_column",
|
|
"type" : {
|
|
"type" : "array",
|
|
"items" : "string"
|
|
}
|
|
}, {
|
|
"name" : "maybe_ints_column",
|
|
"type" : [ "null", {
|
|
"type" : "array",
|
|
"items" : "int"
|
|
} ]
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "AvroArrayOfArray",
|
|
"fields" : [ {
|
|
"name" : "int_arrays_column",
|
|
"type" : {
|
|
"type" : "array",
|
|
"items" : {
|
|
"type" : "array",
|
|
"items" : "int"
|
|
}
|
|
}
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "AvroMapOfArray",
|
|
"fields" : [ {
|
|
"name" : "string_to_ints_column",
|
|
"type" : {
|
|
"type" : "map",
|
|
"values" : {
|
|
"type" : "array",
|
|
"items" : "int"
|
|
}
|
|
}
|
|
} ]
|
|
}, {
|
|
"type" : "record",
|
|
"name" : "ParquetAvroCompat",
|
|
"fields" : [ {
|
|
"name" : "strings_column",
|
|
"type" : {
|
|
"type" : "array",
|
|
"items" : "string"
|
|
}
|
|
}, {
|
|
"name" : "string_to_int_column",
|
|
"type" : {
|
|
"type" : "map",
|
|
"values" : "int"
|
|
}
|
|
}, {
|
|
"name" : "complex_column",
|
|
"type" : {
|
|
"type" : "map",
|
|
"values" : {
|
|
"type" : "array",
|
|
"items" : "Nested"
|
|
}
|
|
}
|
|
} ]
|
|
} ],
|
|
"messages" : { }
|
|
} |