[SPARK-33514][SQL] Migrate TRUNCATE TABLE command to use UnresolvedTable to resolve the identifier
### What changes were proposed in this pull request? This PR proposes to migrate `TRUNCATE TABLE` to use `UnresolvedTable` to resolve the table identifier. This allows consistent resolution rules (temp view first, etc.) to be applied for both v1/v2 commands. More info about the consistent resolution rule proposal can be found in [JIRA](https://issues.apache.org/jira/browse/SPARK-29900) or [proposal doc](https://docs.google.com/document/d/1hvLjGA8y_W_hhilpngXVub1Ebv8RsMap986nENCFnrg/edit?usp=sharing). Note that `TRUNCATE TABLE` works only with v1 tables, and not supported for v2 tables. ### Why are the changes needed? The changes allow consistent resolution behavior when resolving the table identifier. For example, the following is the current behavior: ```scala sql("CREATE TEMPORARY VIEW t AS SELECT 1") sql("CREATE DATABASE db") sql("CREATE TABLE t using csv AS SELECT 1") sql("USE db") sql("TRUNCATE TABLE t") // Succeeds ``` With this PR, `TRUNCATE TABLE` above fails with the following: ``` org.apache.spark.sql.AnalysisException: t is a temp view not table.; line 1 pos 0 at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTempViews$$anonfun$apply$7.$anonfun$applyOrElse$42(Analyzer.scala:866) ``` , which is expected since temporary view is resolved first and `TRUNCATE TABLE` doesn't support a temporary view. ### Does this PR introduce _any_ user-facing change? After this PR, `TRUNCATE TABLE` is resolved to a temp view `t` instead of table `db.t` in the above scenario. ### How was this patch tested? Updated existing tests. Closes #30457 from imback82/truncate_table. Authored-by: Terry Kim <yuminkim@gmail.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
a6555ee596
commit
fdd6c73b3c
|
@ -3356,7 +3356,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a [[TruncateTableStatement]] command.
|
* Create a [[TruncateTable]] command.
|
||||||
*
|
*
|
||||||
* For example:
|
* For example:
|
||||||
* {{{
|
* {{{
|
||||||
|
@ -3364,8 +3364,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
|
||||||
* }}}
|
* }}}
|
||||||
*/
|
*/
|
||||||
override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) {
|
override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) {
|
||||||
TruncateTableStatement(
|
TruncateTable(
|
||||||
visitMultipartIdentifier(ctx.multipartIdentifier),
|
UnresolvedTable(visitMultipartIdentifier(ctx.multipartIdentifier), "TRUNCATE TABLE"),
|
||||||
Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
|
Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -670,3 +670,12 @@ case class LoadData(
|
||||||
case class ShowCreateTable(child: LogicalPlan, asSerde: Boolean = false) extends Command {
|
case class ShowCreateTable(child: LogicalPlan, asSerde: Boolean = false) extends Command {
|
||||||
override def children: Seq[LogicalPlan] = child :: Nil
|
override def children: Seq[LogicalPlan] = child :: Nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The logical plan of the TRUNCATE TABLE command.
|
||||||
|
*/
|
||||||
|
case class TruncateTable(
|
||||||
|
child: LogicalPlan,
|
||||||
|
partitionSpec: Option[TablePartitionSpec]) extends Command {
|
||||||
|
override def children: Seq[LogicalPlan] = child :: Nil
|
||||||
|
}
|
||||||
|
|
|
@ -1621,11 +1621,13 @@ class DDLParserSuite extends AnalysisTest {
|
||||||
test("TRUNCATE table") {
|
test("TRUNCATE table") {
|
||||||
comparePlans(
|
comparePlans(
|
||||||
parsePlan("TRUNCATE TABLE a.b.c"),
|
parsePlan("TRUNCATE TABLE a.b.c"),
|
||||||
TruncateTableStatement(Seq("a", "b", "c"), None))
|
TruncateTable(UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE"), None))
|
||||||
|
|
||||||
comparePlans(
|
comparePlans(
|
||||||
parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"),
|
parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"),
|
||||||
TruncateTableStatement(Seq("a", "b", "c"), Some(Map("ds" -> "2017-06-10"))))
|
TruncateTable(
|
||||||
|
UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE"),
|
||||||
|
Some(Map("ds" -> "2017-06-10"))))
|
||||||
}
|
}
|
||||||
|
|
||||||
test("REFRESH TABLE") {
|
test("REFRESH TABLE") {
|
||||||
|
|
|
@ -456,10 +456,9 @@ class ResolveSessionCatalog(
|
||||||
val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
|
val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
|
||||||
UncacheTableCommand(name.asTableIdentifier, ifExists)
|
UncacheTableCommand(name.asTableIdentifier, ifExists)
|
||||||
|
|
||||||
case TruncateTableStatement(tbl, partitionSpec) =>
|
case TruncateTable(ResolvedV1TableIdentifier(ident), partitionSpec) =>
|
||||||
val v1TableName = parseV1Table(tbl, "TRUNCATE TABLE")
|
|
||||||
TruncateTableCommand(
|
TruncateTableCommand(
|
||||||
v1TableName.asTableIdentifier,
|
ident.asTableIdentifier,
|
||||||
partitionSpec)
|
partitionSpec)
|
||||||
|
|
||||||
case ShowPartitionsStatement(tbl, partitionSpec) =>
|
case ShowPartitionsStatement(tbl, partitionSpec) =>
|
||||||
|
|
|
@ -302,6 +302,9 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
|
||||||
case ShowCreateTable(_: ResolvedTable, _) =>
|
case ShowCreateTable(_: ResolvedTable, _) =>
|
||||||
throw new AnalysisException("SHOW CREATE TABLE is not supported for v2 tables.")
|
throw new AnalysisException("SHOW CREATE TABLE is not supported for v2 tables.")
|
||||||
|
|
||||||
|
case TruncateTable(_: ResolvedTable, _) =>
|
||||||
|
throw new AnalysisException("TRUNCATE TABLE is not supported for v2 tables.")
|
||||||
|
|
||||||
case _ => Nil
|
case _ => Nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1986,8 +1986,8 @@ class DataSourceV2SQLSuite
|
||||||
|PARTITIONED BY (id)
|
|PARTITIONED BY (id)
|
||||||
""".stripMargin)
|
""".stripMargin)
|
||||||
|
|
||||||
testV1Command("TRUNCATE TABLE", t)
|
testNotSupportedV2Command("TRUNCATE TABLE", t)
|
||||||
testV1Command("TRUNCATE TABLE", s"$t PARTITION(id='1')")
|
testNotSupportedV2Command("TRUNCATE TABLE", s"$t PARTITION(id='1')")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -176,15 +176,18 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
|
||||||
sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
|
sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
|
||||||
}.getMessage
|
}.getMessage
|
||||||
assert(e2.contains(s"$viewName is a temp view. 'LOAD DATA' expects a table"))
|
assert(e2.contains(s"$viewName is a temp view. 'LOAD DATA' expects a table"))
|
||||||
assertNoSuchTable(s"TRUNCATE TABLE $viewName")
|
|
||||||
val e3 = intercept[AnalysisException] {
|
val e3 = intercept[AnalysisException] {
|
||||||
|
sql(s"TRUNCATE TABLE $viewName")
|
||||||
|
}.getMessage
|
||||||
|
assert(e3.contains(s"$viewName is a temp view. 'TRUNCATE TABLE' expects a table"))
|
||||||
|
val e4 = intercept[AnalysisException] {
|
||||||
sql(s"SHOW CREATE TABLE $viewName")
|
sql(s"SHOW CREATE TABLE $viewName")
|
||||||
}.getMessage
|
}.getMessage
|
||||||
assert(e3.contains(s"$viewName is a temp view not table or permanent view"))
|
assert(e4.contains(s"$viewName is a temp view not table or permanent view"))
|
||||||
val e4 = intercept[AnalysisException] {
|
val e5 = intercept[AnalysisException] {
|
||||||
sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS")
|
sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS")
|
||||||
}.getMessage
|
}.getMessage
|
||||||
assert(e4.contains(s"$viewName is a temp view not table or permanent view"))
|
assert(e5.contains(s"$viewName is a temp view not table or permanent view"))
|
||||||
assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id")
|
assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -219,7 +222,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
|
||||||
e = intercept[AnalysisException] {
|
e = intercept[AnalysisException] {
|
||||||
sql(s"TRUNCATE TABLE $viewName")
|
sql(s"TRUNCATE TABLE $viewName")
|
||||||
}.getMessage
|
}.getMessage
|
||||||
assert(e.contains(s"Operation not allowed: TRUNCATE TABLE on views: `default`.`testview`"))
|
assert(e.contains("default.testView is a view. 'TRUNCATE TABLE' expects a table"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2169,11 +2169,15 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
|
||||||
(1 to 10).map { i => (i, i) }.toDF("a", "b").createTempView("my_temp_tab")
|
(1 to 10).map { i => (i, i) }.toDF("a", "b").createTempView("my_temp_tab")
|
||||||
sql(s"CREATE TABLE my_ext_tab using parquet LOCATION '${tempDir.toURI}'")
|
sql(s"CREATE TABLE my_ext_tab using parquet LOCATION '${tempDir.toURI}'")
|
||||||
sql(s"CREATE VIEW my_view AS SELECT 1")
|
sql(s"CREATE VIEW my_view AS SELECT 1")
|
||||||
intercept[NoSuchTableException] {
|
val e1 = intercept[AnalysisException] {
|
||||||
sql("TRUNCATE TABLE my_temp_tab")
|
sql("TRUNCATE TABLE my_temp_tab")
|
||||||
}
|
}.getMessage
|
||||||
|
assert(e1.contains("my_temp_tab is a temp view. 'TRUNCATE TABLE' expects a table"))
|
||||||
assertUnsupported("TRUNCATE TABLE my_ext_tab")
|
assertUnsupported("TRUNCATE TABLE my_ext_tab")
|
||||||
assertUnsupported("TRUNCATE TABLE my_view")
|
val e2 = intercept[AnalysisException] {
|
||||||
|
sql("TRUNCATE TABLE my_view")
|
||||||
|
}.getMessage
|
||||||
|
assert(e2.contains("default.my_view is a view. 'TRUNCATE TABLE' expects a table"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue