[SPARK-17112][SQL] "select null" via JDBC triggers IllegalArgumentException in Thriftserver
## What changes were proposed in this pull request? Currently, Spark Thrift Server raises `IllegalArgumentException` for queries whose column types are `NullType`, e.g., `SELECT null` or `SELECT if(true,null,null)`. This PR fixes that by returning `void` like Hive 1.2. **Before** ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ Error: java.lang.IllegalArgumentException: Unrecognized type name: null (state=,code=0) Closing: 0: jdbc:hive2://localhost:10000 $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ Error: java.lang.IllegalArgumentException: Unrecognized type name: null (state=,code=0) Closing: 0: jdbc:hive2://localhost:10000 ``` **After** ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ +-------+--+ | NULL | +-------+--+ | NULL | +-------+--+ 1 row selected (3.242 seconds) Beeline version 1.2.1.spark2 by Apache Hive Closing: 0: jdbc:hive2://localhost:10000 $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" Connecting to jdbc:hive2://localhost:10000 Connected to: Spark SQL (version 2.1.0-SNAPSHOT) Driver: Hive JDBC (version 1.2.1.spark2) Transaction isolation: TRANSACTION_REPEATABLE_READ +-------------------------+--+ | (IF(true, NULL, NULL)) | +-------------------------+--+ | NULL | +-------------------------+--+ 1 row selected (0.201 seconds) Beeline version 1.2.1.spark2 by Apache Hive Closing: 0: jdbc:hive2://localhost:10000 ``` ## How was this patch tested? * Pass the Jenkins test with a new testsuite. * Also, Manually, after starting Spark Thrift Server, run the following command. ```sql $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select null" $ bin/beeline -u jdbc:hive2://localhost:10000 -e "select if(true,null,null)" ``` **Hive 1.2** ```sql hive> create table null_table as select null; hive> desc null_table; OK _c0 void ``` Author: Dongjoon Hyun <dongjoon@apache.org> Closes #15325 from dongjoon-hyun/SPARK-17112.
This commit is contained in:
parent
2bbecdec20
commit
c571cfb2d0
|
@ -56,14 +56,11 @@ private[hive] class SparkExecuteStatementOperation(
|
|||
private var statementId: String = _
|
||||
|
||||
private lazy val resultSchema: TableSchema = {
|
||||
if (result == null || result.queryExecution.analyzed.output.size == 0) {
|
||||
if (result == null || result.schema.isEmpty) {
|
||||
new TableSchema(Arrays.asList(new FieldSchema("Result", "string", "")))
|
||||
} else {
|
||||
logInfo(s"Result Schema: ${result.queryExecution.analyzed.output}")
|
||||
val schema = result.queryExecution.analyzed.output.map { attr =>
|
||||
new FieldSchema(attr.name, attr.dataType.catalogString, "")
|
||||
}
|
||||
new TableSchema(schema.asJava)
|
||||
logInfo(s"Result Schema: ${result.schema}")
|
||||
SparkExecuteStatementOperation.getTableSchema(result.schema)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -282,3 +279,13 @@ private[hive] class SparkExecuteStatementOperation(
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
object SparkExecuteStatementOperation {
|
||||
def getTableSchema(structType: StructType): TableSchema = {
|
||||
val schema = structType.map { field =>
|
||||
val attrTypeString = if (field.dataType == NullType) "void" else field.dataType.catalogString
|
||||
new FieldSchema(field.name, attrTypeString, "")
|
||||
}
|
||||
new TableSchema(schema.asJava)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.spark.sql.hive.thriftserver
|
||||
|
||||
import org.apache.spark.SparkFunSuite
|
||||
import org.apache.spark.sql.types.{NullType, StructField, StructType}
|
||||
|
||||
class SparkExecuteStatementOperationSuite extends SparkFunSuite {
|
||||
test("SPARK-17112 `select null` via JDBC triggers IllegalArgumentException in ThriftServer") {
|
||||
val field1 = StructField("NULL", NullType)
|
||||
val field2 = StructField("(IF(true, NULL, NULL))", NullType)
|
||||
val tableSchema = StructType(Seq(field1, field2))
|
||||
val columns = SparkExecuteStatementOperation.getTableSchema(tableSchema).getColumnDescriptors()
|
||||
assert(columns.size() == 2)
|
||||
assert(columns.get(0).getType() == org.apache.hive.service.cli.Type.NULL_TYPE)
|
||||
assert(columns.get(1).getType() == org.apache.hive.service.cli.Type.NULL_TYPE)
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue