[SPARK-20143][SQL] DataType.fromJson should throw an exception with better message
## What changes were proposed in this pull request? Currently, `DataType.fromJson` throws `scala.MatchError` or `java.util.NoSuchElementException` in some cases when the JSON input is invalid as below: ```scala DataType.fromJson(""""abcd"""") ``` ``` java.util.NoSuchElementException: key not found: abcd at ... ``` ```scala DataType.fromJson("""{"abcd":"a"}""") ``` ``` scala.MatchError: JObject(List((abcd,JString(a)))) (of class org.json4s.JsonAST$JObject) at ... ``` ```scala DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""") ``` ``` scala.MatchError: JObject(List((a,JInt(123)))) (of class org.json4s.JsonAST$JObject) at ... ``` After this PR, ```scala DataType.fromJson(""""abcd"""") ``` ``` java.lang.IllegalArgumentException: Failed to convert the JSON string 'abcd' to a data type. at ... ``` ```scala DataType.fromJson("""{"abcd":"a"}""") ``` ``` java.lang.IllegalArgumentException: Failed to convert the JSON string '{"abcd":"a"}' to a data type. at ... ``` ```scala DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""") at ... ``` ``` java.lang.IllegalArgumentException: Failed to convert the JSON string '{"a":123}' to a field. ``` ## How was this patch tested? Unit test added in `DataTypeSuite`. Author: hyukjinkwon <gurwls223@gmail.com> Closes #17468 from HyukjinKwon/fromjson_exception.
This commit is contained in:
parent
2287f3d0b8
commit
d40cbb8618
|
@ -115,7 +115,10 @@ object DataType {
|
|||
name match {
|
||||
case "decimal" => DecimalType.USER_DEFAULT
|
||||
case FIXED_DECIMAL(precision, scale) => DecimalType(precision.toInt, scale.toInt)
|
||||
case other => nonDecimalNameToType(other)
|
||||
case other => nonDecimalNameToType.getOrElse(
|
||||
other,
|
||||
throw new IllegalArgumentException(
|
||||
s"Failed to convert the JSON string '$name' to a data type."))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -164,6 +167,10 @@ object DataType {
|
|||
("sqlType", v: JValue),
|
||||
("type", JString("udt"))) =>
|
||||
new PythonUserDefinedType(parseDataType(v), pyClass, serialized)
|
||||
|
||||
case other =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Failed to convert the JSON string '${compact(render(other))}' to a data type.")
|
||||
}
|
||||
|
||||
private def parseStructField(json: JValue): StructField = json match {
|
||||
|
@ -179,6 +186,9 @@ object DataType {
|
|||
("nullable", JBool(nullable)),
|
||||
("type", dataType: JValue)) =>
|
||||
StructField(name, parseDataType(dataType), nullable)
|
||||
case other =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Failed to convert the JSON string '${compact(render(other))}' to a field.")
|
||||
}
|
||||
|
||||
protected[types] def buildFormattedString(
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
|
||||
package org.apache.spark.sql.types
|
||||
|
||||
import com.fasterxml.jackson.core.JsonParseException
|
||||
|
||||
import org.apache.spark.{SparkException, SparkFunSuite}
|
||||
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
|
||||
|
||||
|
@ -246,6 +248,32 @@ class DataTypeSuite extends SparkFunSuite {
|
|||
checkDataTypeFromJson(structType)
|
||||
checkDataTypeFromDDL(structType)
|
||||
|
||||
test("fromJson throws an exception when given type string is invalid") {
|
||||
var message = intercept[IllegalArgumentException] {
|
||||
DataType.fromJson(""""abcd"""")
|
||||
}.getMessage
|
||||
assert(message.contains(
|
||||
"Failed to convert the JSON string 'abcd' to a data type."))
|
||||
|
||||
message = intercept[IllegalArgumentException] {
|
||||
DataType.fromJson("""{"abcd":"a"}""")
|
||||
}.getMessage
|
||||
assert(message.contains(
|
||||
"""Failed to convert the JSON string '{"abcd":"a"}' to a data type"""))
|
||||
|
||||
message = intercept[IllegalArgumentException] {
|
||||
DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""")
|
||||
}.getMessage
|
||||
assert(message.contains(
|
||||
"""Failed to convert the JSON string '{"a":123}' to a field."""))
|
||||
|
||||
// Malformed JSON string
|
||||
message = intercept[JsonParseException] {
|
||||
DataType.fromJson("abcd")
|
||||
}.getMessage
|
||||
assert(message.contains("Unrecognized token 'abcd'"))
|
||||
}
|
||||
|
||||
def checkDefaultSize(dataType: DataType, expectedDefaultSize: Int): Unit = {
|
||||
test(s"Check the default size of $dataType") {
|
||||
assert(dataType.defaultSize === expectedDefaultSize)
|
||||
|
|
Loading…
Reference in a new issue