[SPARK-18533] Raise correct error upon specification of schema for datasource tables created using CTAS
## What changes were proposed in this pull request? Fixes the inconsistency of error raised between data source and hive serde tables when schema is specified in CTAS scenario. In the process the grammar for create table (datasource) is simplified. **before:** ``` SQL spark-sql> create table t2 (c1 int, c2 int) using parquet as select * from t1; Error in query: mismatched input 'as' expecting {<EOF>, '.', 'OPTIONS', 'CLUSTERED', 'PARTITIONED'}(line 1, pos 64) == SQL == create table t2 (c1 int, c2 int) using parquet as select * from t1 ----------------------------------------------------------------^^^ ``` **After:** ```SQL spark-sql> create table t2 (c1 int, c2 int) using parquet as select * from t1 > ; Error in query: Operation not allowed: Schema may not be specified in a Create Table As Select (CTAS) statement(line 1, pos 0) == SQL == create table t2 (c1 int, c2 int) using parquet as select * from t1 ^^^ ``` ## How was this patch tested? Added a new test in CreateTableAsSelectSuite Author: Dilip Biswal <dbiswal@us.ibm.com> Closes #15968 from dilipbiswal/ctas.
This commit is contained in:
parent
9c42d4a76c
commit
39a1d30636
|
@ -71,11 +71,7 @@ statement
|
||||||
| createTableHeader ('(' colTypeList ')')? tableProvider
|
| createTableHeader ('(' colTypeList ')')? tableProvider
|
||||||
(OPTIONS tablePropertyList)?
|
(OPTIONS tablePropertyList)?
|
||||||
(PARTITIONED BY partitionColumnNames=identifierList)?
|
(PARTITIONED BY partitionColumnNames=identifierList)?
|
||||||
bucketSpec? #createTableUsing
|
bucketSpec? (AS? query)? #createTableUsing
|
||||||
| createTableHeader tableProvider
|
|
||||||
(OPTIONS tablePropertyList)?
|
|
||||||
(PARTITIONED BY partitionColumnNames=identifierList)?
|
|
||||||
bucketSpec? AS? query #createTableUsing
|
|
||||||
| createTableHeader ('(' columns=colTypeList ')')?
|
| createTableHeader ('(' columns=colTypeList ')')?
|
||||||
(COMMENT STRING)?
|
(COMMENT STRING)?
|
||||||
(PARTITIONED BY '(' partitionColumns=colTypeList ')')?
|
(PARTITIONED BY '(' partitionColumns=colTypeList ')')?
|
||||||
|
|
|
@ -322,7 +322,20 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a [[CreateTable]] logical plan.
|
* Create a data source table, returning a [[CreateTable]] logical plan.
|
||||||
|
*
|
||||||
|
* Expected format:
|
||||||
|
* {{{
|
||||||
|
* CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [db_name.]table_name
|
||||||
|
* USING table_provider
|
||||||
|
* [OPTIONS table_property_list]
|
||||||
|
* [PARTITIONED BY (col_name, col_name, ...)]
|
||||||
|
* [CLUSTERED BY (col_name, col_name, ...)
|
||||||
|
* [SORTED BY (col_name [ASC|DESC], ...)]
|
||||||
|
* INTO num_buckets BUCKETS
|
||||||
|
* ]
|
||||||
|
* [AS select_statement];
|
||||||
|
* }}}
|
||||||
*/
|
*/
|
||||||
override def visitCreateTableUsing(ctx: CreateTableUsingContext): LogicalPlan = withOrigin(ctx) {
|
override def visitCreateTableUsing(ctx: CreateTableUsingContext): LogicalPlan = withOrigin(ctx) {
|
||||||
val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader)
|
val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader)
|
||||||
|
@ -371,6 +384,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
|
||||||
operationNotAllowed("CREATE TEMPORARY TABLE ... USING ... AS query", ctx)
|
operationNotAllowed("CREATE TEMPORARY TABLE ... USING ... AS query", ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Don't allow explicit specification of schema for CTAS
|
||||||
|
if (schema.nonEmpty) {
|
||||||
|
operationNotAllowed(
|
||||||
|
"Schema may not be specified in a Create Table As Select (CTAS) statement",
|
||||||
|
ctx)
|
||||||
|
}
|
||||||
CreateTable(tableDesc, mode, Some(query))
|
CreateTable(tableDesc, mode, Some(query))
|
||||||
} else {
|
} else {
|
||||||
if (temp) {
|
if (temp) {
|
||||||
|
@ -1052,7 +1071,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
|
||||||
"CTAS statement."
|
"CTAS statement."
|
||||||
operationNotAllowed(errorMessage, ctx)
|
operationNotAllowed(errorMessage, ctx)
|
||||||
}
|
}
|
||||||
// Just use whatever is projected in the select statement as our schema
|
|
||||||
|
// Don't allow explicit specification of schema for CTAS.
|
||||||
if (schema.nonEmpty) {
|
if (schema.nonEmpty) {
|
||||||
operationNotAllowed(
|
operationNotAllowed(
|
||||||
"Schema may not be specified in a Create Table As Select (CTAS) statement",
|
"Schema may not be specified in a Create Table As Select (CTAS) statement",
|
||||||
|
|
|
@ -249,4 +249,13 @@ class CreateTableAsSelectSuite
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test("specifying the column list for CTAS") {
|
||||||
|
withTable("t") {
|
||||||
|
val e = intercept[ParseException] {
|
||||||
|
sql("CREATE TABLE t (a int, b int) USING parquet AS SELECT 1, 2")
|
||||||
|
}.getMessage
|
||||||
|
assert(e.contains("Schema may not be specified in a Create Table As Select (CTAS)"))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue