[SPARK-33514][SQL] Migrate TRUNCATE TABLE command to use UnresolvedTable to resolve the identifier

### What changes were proposed in this pull request?

This PR proposes to migrate `TRUNCATE TABLE` to use `UnresolvedTable` to resolve the table identifier. This allows consistent resolution rules (temp view first, etc.) to be applied for both v1/v2 commands. More info about the consistent resolution rule proposal can be found in [JIRA](https://issues.apache.org/jira/browse/SPARK-29900) or [proposal doc](https://docs.google.com/document/d/1hvLjGA8y_W_hhilpngXVub1Ebv8RsMap986nENCFnrg/edit?usp=sharing).

Note that `TRUNCATE TABLE` works only with v1 tables, and not supported for v2 tables.

### Why are the changes needed?

The changes allow consistent resolution behavior when resolving the table identifier. For example, the following is the current behavior:
```scala
sql("CREATE TEMPORARY VIEW t AS SELECT 1")
sql("CREATE DATABASE db")
sql("CREATE TABLE t using csv AS SELECT 1")
sql("USE db")
sql("TRUNCATE TABLE t") // Succeeds
```
With this PR, `TRUNCATE TABLE` above fails with the following:
```
org.apache.spark.sql.AnalysisException: t is a temp view not table.; line 1 pos 0
    at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
    at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTempViews$$anonfun$apply$7.$anonfun$applyOrElse$42(Analyzer.scala:866)

```
, which is expected since temporary view is resolved first and `TRUNCATE TABLE` doesn't support a temporary view.

### Does this PR introduce _any_ user-facing change?

After this PR, `TRUNCATE TABLE` is resolved to a temp view `t` instead of table `db.t` in the above scenario.

### How was this patch tested?

Updated existing tests.

Closes #30457 from imback82/truncate_table.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
Terry Kim 2020-11-24 11:06:39 +00:00 committed by Wenchen Fan
parent a6555ee596
commit fdd6c73b3c
8 changed files with 38 additions and 18 deletions

View file

@ -3356,7 +3356,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
} }
/** /**
* Create a [[TruncateTableStatement]] command. * Create a [[TruncateTable]] command.
* *
* For example: * For example:
* {{{ * {{{
@ -3364,8 +3364,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
* }}} * }}}
*/ */
override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) { override def visitTruncateTable(ctx: TruncateTableContext): LogicalPlan = withOrigin(ctx) {
TruncateTableStatement( TruncateTable(
visitMultipartIdentifier(ctx.multipartIdentifier), UnresolvedTable(visitMultipartIdentifier(ctx.multipartIdentifier), "TRUNCATE TABLE"),
Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)) Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
} }

View file

@ -670,3 +670,12 @@ case class LoadData(
case class ShowCreateTable(child: LogicalPlan, asSerde: Boolean = false) extends Command { case class ShowCreateTable(child: LogicalPlan, asSerde: Boolean = false) extends Command {
override def children: Seq[LogicalPlan] = child :: Nil override def children: Seq[LogicalPlan] = child :: Nil
} }
/**
* The logical plan of the TRUNCATE TABLE command.
*/
case class TruncateTable(
child: LogicalPlan,
partitionSpec: Option[TablePartitionSpec]) extends Command {
override def children: Seq[LogicalPlan] = child :: Nil
}

View file

@ -1621,11 +1621,13 @@ class DDLParserSuite extends AnalysisTest {
test("TRUNCATE table") { test("TRUNCATE table") {
comparePlans( comparePlans(
parsePlan("TRUNCATE TABLE a.b.c"), parsePlan("TRUNCATE TABLE a.b.c"),
TruncateTableStatement(Seq("a", "b", "c"), None)) TruncateTable(UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE"), None))
comparePlans( comparePlans(
parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"), parsePlan("TRUNCATE TABLE a.b.c PARTITION(ds='2017-06-10')"),
TruncateTableStatement(Seq("a", "b", "c"), Some(Map("ds" -> "2017-06-10")))) TruncateTable(
UnresolvedTable(Seq("a", "b", "c"), "TRUNCATE TABLE"),
Some(Map("ds" -> "2017-06-10"))))
} }
test("REFRESH TABLE") { test("REFRESH TABLE") {

View file

@ -456,10 +456,9 @@ class ResolveSessionCatalog(
val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE") val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
UncacheTableCommand(name.asTableIdentifier, ifExists) UncacheTableCommand(name.asTableIdentifier, ifExists)
case TruncateTableStatement(tbl, partitionSpec) => case TruncateTable(ResolvedV1TableIdentifier(ident), partitionSpec) =>
val v1TableName = parseV1Table(tbl, "TRUNCATE TABLE")
TruncateTableCommand( TruncateTableCommand(
v1TableName.asTableIdentifier, ident.asTableIdentifier,
partitionSpec) partitionSpec)
case ShowPartitionsStatement(tbl, partitionSpec) => case ShowPartitionsStatement(tbl, partitionSpec) =>

View file

@ -302,6 +302,9 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
case ShowCreateTable(_: ResolvedTable, _) => case ShowCreateTable(_: ResolvedTable, _) =>
throw new AnalysisException("SHOW CREATE TABLE is not supported for v2 tables.") throw new AnalysisException("SHOW CREATE TABLE is not supported for v2 tables.")
case TruncateTable(_: ResolvedTable, _) =>
throw new AnalysisException("TRUNCATE TABLE is not supported for v2 tables.")
case _ => Nil case _ => Nil
} }
} }

View file

@ -1986,8 +1986,8 @@ class DataSourceV2SQLSuite
|PARTITIONED BY (id) |PARTITIONED BY (id)
""".stripMargin) """.stripMargin)
testV1Command("TRUNCATE TABLE", t) testNotSupportedV2Command("TRUNCATE TABLE", t)
testV1Command("TRUNCATE TABLE", s"$t PARTITION(id='1')") testNotSupportedV2Command("TRUNCATE TABLE", s"$t PARTITION(id='1')")
} }
} }

View file

@ -176,15 +176,18 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""") sql(s"""LOAD DATA LOCAL INPATH "$dataFilePath" INTO TABLE $viewName""")
}.getMessage }.getMessage
assert(e2.contains(s"$viewName is a temp view. 'LOAD DATA' expects a table")) assert(e2.contains(s"$viewName is a temp view. 'LOAD DATA' expects a table"))
assertNoSuchTable(s"TRUNCATE TABLE $viewName")
val e3 = intercept[AnalysisException] { val e3 = intercept[AnalysisException] {
sql(s"TRUNCATE TABLE $viewName")
}.getMessage
assert(e3.contains(s"$viewName is a temp view. 'TRUNCATE TABLE' expects a table"))
val e4 = intercept[AnalysisException] {
sql(s"SHOW CREATE TABLE $viewName") sql(s"SHOW CREATE TABLE $viewName")
}.getMessage }.getMessage
assert(e3.contains(s"$viewName is a temp view not table or permanent view")) assert(e4.contains(s"$viewName is a temp view not table or permanent view"))
val e4 = intercept[AnalysisException] { val e5 = intercept[AnalysisException] {
sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS") sql(s"ANALYZE TABLE $viewName COMPUTE STATISTICS")
}.getMessage }.getMessage
assert(e4.contains(s"$viewName is a temp view not table or permanent view")) assert(e5.contains(s"$viewName is a temp view not table or permanent view"))
assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id") assertNoSuchTable(s"ANALYZE TABLE $viewName COMPUTE STATISTICS FOR COLUMNS id")
} }
} }
@ -219,7 +222,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
e = intercept[AnalysisException] { e = intercept[AnalysisException] {
sql(s"TRUNCATE TABLE $viewName") sql(s"TRUNCATE TABLE $viewName")
}.getMessage }.getMessage
assert(e.contains(s"Operation not allowed: TRUNCATE TABLE on views: `default`.`testview`")) assert(e.contains("default.testView is a view. 'TRUNCATE TABLE' expects a table"))
} }
} }

View file

@ -2169,11 +2169,15 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
(1 to 10).map { i => (i, i) }.toDF("a", "b").createTempView("my_temp_tab") (1 to 10).map { i => (i, i) }.toDF("a", "b").createTempView("my_temp_tab")
sql(s"CREATE TABLE my_ext_tab using parquet LOCATION '${tempDir.toURI}'") sql(s"CREATE TABLE my_ext_tab using parquet LOCATION '${tempDir.toURI}'")
sql(s"CREATE VIEW my_view AS SELECT 1") sql(s"CREATE VIEW my_view AS SELECT 1")
intercept[NoSuchTableException] { val e1 = intercept[AnalysisException] {
sql("TRUNCATE TABLE my_temp_tab") sql("TRUNCATE TABLE my_temp_tab")
} }.getMessage
assert(e1.contains("my_temp_tab is a temp view. 'TRUNCATE TABLE' expects a table"))
assertUnsupported("TRUNCATE TABLE my_ext_tab") assertUnsupported("TRUNCATE TABLE my_ext_tab")
assertUnsupported("TRUNCATE TABLE my_view") val e2 = intercept[AnalysisException] {
sql("TRUNCATE TABLE my_view")
}.getMessage
assert(e2.contains("default.my_view is a view. 'TRUNCATE TABLE' expects a table"))
} }
} }
} }