[SPARK-35085][SQL] Get columns operation should handle ANSI interval column properly
### What changes were proposed in this pull request? This PR let JDBC clients identify ANSI interval columns properly. ### Why are the changes needed? This PR is similar to https://github.com/apache/spark/pull/29539. JDBC users can query interval values through thrift server, create views with ansi interval columns, e.g. `CREATE global temp view view1 as select interval '1-1' year to month as I;` but when they want to get the details of the columns of view1, the will fail with `Unrecognized type name: YEAR-MONTH INTERVAL` ``` Caused by: java.lang.IllegalArgumentException: Unrecognized type name: YEAR-MONTH INTERVAL at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.toJavaSQLType(SparkGetColumnsOperation.scala:190) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.$anonfun$addToRowSet$1(SparkGetColumnsOperation.scala:206) at scala.collection.immutable.List.foreach(List.scala:392) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.addToRowSet(SparkGetColumnsOperation.scala:198) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.$anonfun$runInternal$7(SparkGetColumnsOperation.scala:109) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.$anonfun$runInternal$7$adapted(SparkGetColumnsOperation.scala:109) at scala.Option.foreach(Option.scala:407) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.$anonfun$runInternal$5(SparkGetColumnsOperation.scala:109) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.$anonfun$runInternal$5$adapted(SparkGetColumnsOperation.scala:107) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.sql.hive.thriftserver.SparkGetColumnsOperation.runInternal(SparkGetColumnsOperation.scala:107) ... 34 more ``` ### Does this PR introduce _any_ user-facing change? Yes. Let hive JDBC recognize ANSI interval. ### How was this patch tested? Jenkins test. Closes #32345 from beliefer/SPARK-35085. Lead-authored-by: gengjiaan <gengjiaan@360.cn> Co-authored-by: beliefer <beliefer@163.com> Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
046c8c3dd6
commit
56bb8155c5
|
@ -131,7 +131,7 @@ private[hive] class SparkGetColumnsOperation(
|
||||||
*/
|
*/
|
||||||
private def getColumnSize(typ: DataType): Option[Int] = typ match {
|
private def getColumnSize(typ: DataType): Option[Int] = typ match {
|
||||||
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
|
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
|
||||||
CalendarIntervalType | NullType) =>
|
CalendarIntervalType | NullType | YearMonthIntervalType | DayTimeIntervalType) =>
|
||||||
Some(dt.defaultSize)
|
Some(dt.defaultSize)
|
||||||
case CharType(n) => Some(n)
|
case CharType(n) => Some(n)
|
||||||
case StructType(fields) =>
|
case StructType(fields) =>
|
||||||
|
@ -186,7 +186,8 @@ private[hive] class SparkGetColumnsOperation(
|
||||||
case _: MapType => java.sql.Types.JAVA_OBJECT
|
case _: MapType => java.sql.Types.JAVA_OBJECT
|
||||||
case _: StructType => java.sql.Types.STRUCT
|
case _: StructType => java.sql.Types.STRUCT
|
||||||
// Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
|
// Hive's year-month and day-time intervals are mapping to java.sql.Types.OTHER
|
||||||
case _: CalendarIntervalType => java.sql.Types.OTHER
|
case _: CalendarIntervalType | YearMonthIntervalType | DayTimeIntervalType =>
|
||||||
|
java.sql.Types.OTHER
|
||||||
case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
|
case _ => throw new IllegalArgumentException(s"Unrecognized type name: ${typ.sql}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -377,6 +377,60 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test("SPARK-35085: Get columns operation should handle ANSI interval column properly") {
|
||||||
|
val viewName1 = "view_interval1"
|
||||||
|
val yearMonthDDL =
|
||||||
|
s"CREATE GLOBAL TEMP VIEW $viewName1 as select interval '1-1' year to month as i"
|
||||||
|
|
||||||
|
withJdbcStatement(viewName1) { statement =>
|
||||||
|
statement.execute(yearMonthDDL)
|
||||||
|
val data = statement.getConnection.getMetaData
|
||||||
|
val rowSet = data.getColumns("", "global_temp", viewName1, null)
|
||||||
|
while (rowSet.next()) {
|
||||||
|
assert(rowSet.getString("TABLE_CAT") === null)
|
||||||
|
assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
|
||||||
|
assert(rowSet.getString("TABLE_NAME") === viewName1)
|
||||||
|
assert(rowSet.getString("COLUMN_NAME") === "i")
|
||||||
|
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
|
||||||
|
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(YearMonthIntervalType.sql))
|
||||||
|
assert(rowSet.getInt("COLUMN_SIZE") === YearMonthIntervalType.defaultSize)
|
||||||
|
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
|
||||||
|
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
|
||||||
|
assert(rowSet.getInt("NULLABLE") === 0)
|
||||||
|
assert(rowSet.getString("REMARKS") === "")
|
||||||
|
assert(rowSet.getInt("ORDINAL_POSITION") === 0)
|
||||||
|
assert(rowSet.getString("IS_NULLABLE") === "YES")
|
||||||
|
assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val viewName2 = "view_interval2"
|
||||||
|
val dayTimeDDL =
|
||||||
|
s"CREATE GLOBAL TEMP VIEW $viewName2 as select interval '1 2:3:4.001' day to second as i"
|
||||||
|
|
||||||
|
withJdbcStatement(viewName2) { statement =>
|
||||||
|
statement.execute(dayTimeDDL)
|
||||||
|
val data = statement.getConnection.getMetaData
|
||||||
|
val rowSet = data.getColumns("", "global_temp", viewName2, null)
|
||||||
|
while (rowSet.next()) {
|
||||||
|
assert(rowSet.getString("TABLE_CAT") === null)
|
||||||
|
assert(rowSet.getString("TABLE_SCHEM") === "global_temp")
|
||||||
|
assert(rowSet.getString("TABLE_NAME") === viewName2)
|
||||||
|
assert(rowSet.getString("COLUMN_NAME") === "i")
|
||||||
|
assert(rowSet.getInt("DATA_TYPE") === java.sql.Types.OTHER)
|
||||||
|
assert(rowSet.getString("TYPE_NAME").equalsIgnoreCase(DayTimeIntervalType.sql))
|
||||||
|
assert(rowSet.getInt("COLUMN_SIZE") === DayTimeIntervalType.defaultSize)
|
||||||
|
assert(rowSet.getInt("DECIMAL_DIGITS") === 0)
|
||||||
|
assert(rowSet.getInt("NUM_PREC_RADIX") === 0)
|
||||||
|
assert(rowSet.getInt("NULLABLE") === 0)
|
||||||
|
assert(rowSet.getString("REMARKS") === "")
|
||||||
|
assert(rowSet.getInt("ORDINAL_POSITION") === 0)
|
||||||
|
assert(rowSet.getString("IS_NULLABLE") === "YES")
|
||||||
|
assert(rowSet.getString("IS_AUTO_INCREMENT") === "NO")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
test("handling null in view for get columns operations") {
|
test("handling null in view for get columns operations") {
|
||||||
val viewName = "view_null"
|
val viewName = "view_null"
|
||||||
val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select null as n"
|
val ddl = s"CREATE GLOBAL TEMP VIEW $viewName as select null as n"
|
||||||
|
|
Loading…
Reference in a new issue