[SPARK-18961][SQL] Support SHOW TABLE EXTENDED ... PARTITION
statement
## What changes were proposed in this pull request? We should support the statement `SHOW TABLE EXTENDED LIKE 'table_identifier' PARTITION(partition_spec)`, just like that HIVE does. When partition is specified, the `SHOW TABLE EXTENDED` command should output the information of the partitions instead of the tables. Note that in this statement, we require exact matched partition spec. For example: ``` CREATE TABLE show_t1(a String, b Int) PARTITIONED BY (c String, d String); ALTER TABLE show_t1 ADD PARTITION (c='Us', d=1) PARTITION (c='Us', d=22); -- Output the extended information of Partition(c='Us', d=1) SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1); -- Throw an AnalysisException SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us'); ``` ## How was this patch tested? Add new test sqls in file `show-tables.sql`. Add new test case in `DDLSuite`. Author: jiangxingbo <jiangxb1987@gmail.com> Closes #16373 from jiangxb1987/show-partition-extended.
This commit is contained in:
parent
85941ecf28
commit
a02a0b1703
|
@ -127,8 +127,8 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
|
||||||
.map(s => String.format(s"%-20s", s))
|
.map(s => String.format(s"%-20s", s))
|
||||||
.mkString("\t")
|
.mkString("\t")
|
||||||
}
|
}
|
||||||
// SHOW TABLES in Hive only output table names, while ours outputs database, table name, isTemp.
|
// SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
|
||||||
case command: ExecutedCommandExec if command.cmd.isInstanceOf[ShowTablesCommand] =>
|
case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
|
||||||
command.executeCollect().map(_.getString(1))
|
command.executeCollect().map(_.getString(1))
|
||||||
case other =>
|
case other =>
|
||||||
val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
|
val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
|
||||||
|
|
|
@ -134,7 +134,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
|
||||||
ShowTablesCommand(
|
ShowTablesCommand(
|
||||||
Option(ctx.db).map(_.getText),
|
Option(ctx.db).map(_.getText),
|
||||||
Option(ctx.pattern).map(string),
|
Option(ctx.pattern).map(string),
|
||||||
isExtended = false)
|
isExtended = false,
|
||||||
|
partitionSpec = None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -146,14 +147,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
|
||||||
* }}}
|
* }}}
|
||||||
*/
|
*/
|
||||||
override def visitShowTable(ctx: ShowTableContext): LogicalPlan = withOrigin(ctx) {
|
override def visitShowTable(ctx: ShowTableContext): LogicalPlan = withOrigin(ctx) {
|
||||||
if (ctx.partitionSpec != null) {
|
val partitionSpec = Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)
|
||||||
operationNotAllowed("SHOW TABLE EXTENDED ... PARTITION", ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
ShowTablesCommand(
|
ShowTablesCommand(
|
||||||
Option(ctx.db).map(_.getText),
|
Option(ctx.db).map(_.getText),
|
||||||
Option(ctx.pattern).map(string),
|
Option(ctx.pattern).map(string),
|
||||||
isExtended = true)
|
isExtended = true,
|
||||||
|
partitionSpec = partitionSpec)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -616,13 +616,15 @@ case class DescribeTableCommand(
|
||||||
* The syntax of using this command in SQL is:
|
* The syntax of using this command in SQL is:
|
||||||
* {{{
|
* {{{
|
||||||
* SHOW TABLES [(IN|FROM) database_name] [[LIKE] 'identifier_with_wildcards'];
|
* SHOW TABLES [(IN|FROM) database_name] [[LIKE] 'identifier_with_wildcards'];
|
||||||
* SHOW TABLE EXTENDED [(IN|FROM) database_name] LIKE 'identifier_with_wildcards';
|
* SHOW TABLE EXTENDED [(IN|FROM) database_name] LIKE 'identifier_with_wildcards'
|
||||||
|
* [PARTITION(partition_spec)];
|
||||||
* }}}
|
* }}}
|
||||||
*/
|
*/
|
||||||
case class ShowTablesCommand(
|
case class ShowTablesCommand(
|
||||||
databaseName: Option[String],
|
databaseName: Option[String],
|
||||||
tableIdentifierPattern: Option[String],
|
tableIdentifierPattern: Option[String],
|
||||||
isExtended: Boolean = false) extends RunnableCommand {
|
isExtended: Boolean = false,
|
||||||
|
partitionSpec: Option[TablePartitionSpec] = None) extends RunnableCommand {
|
||||||
|
|
||||||
// The result of SHOW TABLES/SHOW TABLE has three basic columns: database, tableName and
|
// The result of SHOW TABLES/SHOW TABLE has three basic columns: database, tableName and
|
||||||
// isTemporary. If `isExtended` is true, append column `information` to the output columns.
|
// isTemporary. If `isExtended` is true, append column `information` to the output columns.
|
||||||
|
@ -642,6 +644,8 @@ case class ShowTablesCommand(
|
||||||
// instead of calling tables in sparkSession.
|
// instead of calling tables in sparkSession.
|
||||||
val catalog = sparkSession.sessionState.catalog
|
val catalog = sparkSession.sessionState.catalog
|
||||||
val db = databaseName.getOrElse(catalog.getCurrentDatabase)
|
val db = databaseName.getOrElse(catalog.getCurrentDatabase)
|
||||||
|
if (partitionSpec.isEmpty) {
|
||||||
|
// Show the information of tables.
|
||||||
val tables =
|
val tables =
|
||||||
tableIdentifierPattern.map(catalog.listTables(db, _)).getOrElse(catalog.listTables(db))
|
tableIdentifierPattern.map(catalog.listTables(db, _)).getOrElse(catalog.listTables(db))
|
||||||
tables.map { tableIdent =>
|
tables.map { tableIdent =>
|
||||||
|
@ -650,11 +654,25 @@ case class ShowTablesCommand(
|
||||||
val isTemp = catalog.isTemporaryTable(tableIdent)
|
val isTemp = catalog.isTemporaryTable(tableIdent)
|
||||||
if (isExtended) {
|
if (isExtended) {
|
||||||
val information = catalog.getTempViewOrPermanentTableMetadata(tableIdent).toString
|
val information = catalog.getTempViewOrPermanentTableMetadata(tableIdent).toString
|
||||||
Row(database, tableName, isTemp, s"${information}\n")
|
Row(database, tableName, isTemp, s"$information\n")
|
||||||
} else {
|
} else {
|
||||||
Row(database, tableName, isTemp)
|
Row(database, tableName, isTemp)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// Show the information of partitions.
|
||||||
|
//
|
||||||
|
// Note: tableIdentifierPattern should be non-empty, otherwise a [[ParseException]]
|
||||||
|
// should have been thrown by the sql parser.
|
||||||
|
val tableIdent = TableIdentifier(tableIdentifierPattern.get, Some(db))
|
||||||
|
val table = catalog.getTableMetadata(tableIdent).identifier
|
||||||
|
val partition = catalog.getPartition(tableIdent, partitionSpec.get)
|
||||||
|
val database = table.database.getOrElse("")
|
||||||
|
val tableName = table.table
|
||||||
|
val isTemp = catalog.isTemporaryTable(table)
|
||||||
|
val information = partition.toString
|
||||||
|
Seq(Row(database, tableName, isTemp, s"$information\n"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,10 +17,21 @@ SHOW TABLES LIKE 'show_t1*|show_t2*';
|
||||||
SHOW TABLES IN showdb 'show_t*';
|
SHOW TABLES IN showdb 'show_t*';
|
||||||
|
|
||||||
-- SHOW TABLE EXTENDED
|
-- SHOW TABLE EXTENDED
|
||||||
-- Ignore these because there exist timestamp results, e.g. `Created`.
|
SHOW TABLE EXTENDED LIKE 'show_t*';
|
||||||
-- SHOW TABLE EXTENDED LIKE 'show_t*';
|
|
||||||
SHOW TABLE EXTENDED;
|
SHOW TABLE EXTENDED;
|
||||||
|
|
||||||
|
-- SHOW TABLE EXTENDED ... PARTITION
|
||||||
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1);
|
||||||
|
-- Throw a ParseException if table name is not specified.
|
||||||
|
SHOW TABLE EXTENDED PARTITION(c='Us', d=1);
|
||||||
|
-- Don't support regular expression for table name if a partition specification is present.
|
||||||
|
SHOW TABLE EXTENDED LIKE 'show_t*' PARTITION(c='Us', d=1);
|
||||||
|
-- Partition specification is not complete.
|
||||||
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us');
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us');
|
||||||
|
-- Partition specification is invalid.
|
||||||
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(a='Us', d=1);
|
||||||
|
-- Partition specification doesn't exist.
|
||||||
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Ch', d=1);
|
||||||
|
|
||||||
-- Clean Up
|
-- Clean Up
|
||||||
DROP TABLE show_t1;
|
DROP TABLE show_t1;
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
-- Automatically generated by SQLQueryTestSuite
|
-- Automatically generated by SQLQueryTestSuite
|
||||||
-- Number of queries: 20
|
-- Number of queries: 26
|
||||||
|
|
||||||
|
|
||||||
-- !query 0
|
-- !query 0
|
||||||
|
@ -114,10 +114,44 @@ show_t3
|
||||||
|
|
||||||
|
|
||||||
-- !query 12
|
-- !query 12
|
||||||
SHOW TABLE EXTENDED
|
SHOW TABLE EXTENDED LIKE 'show_t*'
|
||||||
-- !query 12 schema
|
-- !query 12 schema
|
||||||
struct<>
|
struct<database:string,tableName:string,isTemporary:boolean,information:string>
|
||||||
-- !query 12 output
|
-- !query 12 output
|
||||||
|
show_t3 true CatalogTable(
|
||||||
|
Table: `show_t3`
|
||||||
|
Created:
|
||||||
|
Last Access:
|
||||||
|
Type: VIEW
|
||||||
|
Schema: [StructField(e,IntegerType,true)]
|
||||||
|
Storage())
|
||||||
|
|
||||||
|
showdb show_t1 false CatalogTable(
|
||||||
|
Table: `showdb`.`show_t1`
|
||||||
|
Created:
|
||||||
|
Last Access:
|
||||||
|
Type: MANAGED
|
||||||
|
Schema: [StructField(a,StringType,true), StructField(b,IntegerType,true), StructField(c,StringType,true), StructField(d,StringType,true)]
|
||||||
|
Provider: parquet
|
||||||
|
Partition Columns: [`c`, `d`]
|
||||||
|
Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1)
|
||||||
|
Partition Provider: Catalog)
|
||||||
|
|
||||||
|
showdb show_t2 false CatalogTable(
|
||||||
|
Table: `showdb`.`show_t2`
|
||||||
|
Created:
|
||||||
|
Last Access:
|
||||||
|
Type: MANAGED
|
||||||
|
Schema: [StructField(b,StringType,true), StructField(d,IntegerType,true)]
|
||||||
|
Provider: parquet
|
||||||
|
Storage(Location: sql/core/spark-warehouse/showdb.db/show_t2))
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 13
|
||||||
|
SHOW TABLE EXTENDED
|
||||||
|
-- !query 13 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 13 output
|
||||||
org.apache.spark.sql.catalyst.parser.ParseException
|
org.apache.spark.sql.catalyst.parser.ParseException
|
||||||
|
|
||||||
mismatched input '<EOF>' expecting 'LIKE'(line 1, pos 19)
|
mismatched input '<EOF>' expecting 'LIKE'(line 1, pos 19)
|
||||||
|
@ -127,63 +161,112 @@ SHOW TABLE EXTENDED
|
||||||
-------------------^^^
|
-------------------^^^
|
||||||
|
|
||||||
|
|
||||||
-- !query 13
|
|
||||||
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us')
|
|
||||||
-- !query 13 schema
|
|
||||||
struct<>
|
|
||||||
-- !query 13 output
|
|
||||||
org.apache.spark.sql.catalyst.parser.ParseException
|
|
||||||
|
|
||||||
Operation not allowed: SHOW TABLE EXTENDED ... PARTITION(line 1, pos 0)
|
|
||||||
|
|
||||||
== SQL ==
|
|
||||||
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us')
|
|
||||||
^^^
|
|
||||||
|
|
||||||
|
|
||||||
-- !query 14
|
-- !query 14
|
||||||
DROP TABLE show_t1
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us', d=1)
|
||||||
-- !query 14 schema
|
-- !query 14 schema
|
||||||
struct<>
|
struct<database:string,tableName:string,isTemporary:boolean,information:string>
|
||||||
-- !query 14 output
|
-- !query 14 output
|
||||||
|
showdb show_t1 false CatalogPartition(
|
||||||
|
Partition Values: [c=Us, d=1]
|
||||||
|
Storage(Location: sql/core/spark-warehouse/showdb.db/show_t1/c=Us/d=1)
|
||||||
|
Partition Parameters:{})
|
||||||
|
|
||||||
|
|
||||||
-- !query 15
|
-- !query 15
|
||||||
DROP TABLE show_t2
|
SHOW TABLE EXTENDED PARTITION(c='Us', d=1)
|
||||||
-- !query 15 schema
|
-- !query 15 schema
|
||||||
struct<>
|
struct<>
|
||||||
-- !query 15 output
|
-- !query 15 output
|
||||||
|
org.apache.spark.sql.catalyst.parser.ParseException
|
||||||
|
|
||||||
|
mismatched input 'PARTITION' expecting 'LIKE'(line 1, pos 20)
|
||||||
|
|
||||||
|
== SQL ==
|
||||||
|
SHOW TABLE EXTENDED PARTITION(c='Us', d=1)
|
||||||
|
--------------------^^^
|
||||||
|
|
||||||
|
|
||||||
-- !query 16
|
-- !query 16
|
||||||
DROP VIEW show_t3
|
SHOW TABLE EXTENDED LIKE 'show_t*' PARTITION(c='Us', d=1)
|
||||||
-- !query 16 schema
|
-- !query 16 schema
|
||||||
struct<>
|
struct<>
|
||||||
-- !query 16 output
|
-- !query 16 output
|
||||||
|
org.apache.spark.sql.catalyst.analysis.NoSuchTableException
|
||||||
|
Table or view 'show_t*' not found in database 'showdb';
|
||||||
|
|
||||||
|
|
||||||
-- !query 17
|
-- !query 17
|
||||||
DROP VIEW global_temp.show_t4
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Us')
|
||||||
-- !query 17 schema
|
-- !query 17 schema
|
||||||
struct<>
|
struct<>
|
||||||
-- !query 17 output
|
-- !query 17 output
|
||||||
|
org.apache.spark.sql.AnalysisException
|
||||||
|
Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
|
||||||
|
|
||||||
|
|
||||||
-- !query 18
|
-- !query 18
|
||||||
USE default
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(a='Us', d=1)
|
||||||
-- !query 18 schema
|
-- !query 18 schema
|
||||||
struct<>
|
struct<>
|
||||||
-- !query 18 output
|
-- !query 18 output
|
||||||
|
org.apache.spark.sql.AnalysisException
|
||||||
|
Partition spec is invalid. The spec (a, d) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
|
||||||
|
|
||||||
|
|
||||||
-- !query 19
|
-- !query 19
|
||||||
DROP DATABASE showdb
|
SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(c='Ch', d=1)
|
||||||
-- !query 19 schema
|
-- !query 19 schema
|
||||||
struct<>
|
struct<>
|
||||||
-- !query 19 output
|
-- !query 19 output
|
||||||
|
org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
|
||||||
|
Partition not found in table 'show_t1' database 'showdb':
|
||||||
|
c -> Ch
|
||||||
|
d -> 1;
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 20
|
||||||
|
DROP TABLE show_t1
|
||||||
|
-- !query 20 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 20 output
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 21
|
||||||
|
DROP TABLE show_t2
|
||||||
|
-- !query 21 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 21 output
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 22
|
||||||
|
DROP VIEW show_t3
|
||||||
|
-- !query 22 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 22 output
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 23
|
||||||
|
DROP VIEW global_temp.show_t4
|
||||||
|
-- !query 23 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 23 output
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 24
|
||||||
|
USE default
|
||||||
|
-- !query 24 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 24 output
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- !query 25
|
||||||
|
DROP DATABASE showdb
|
||||||
|
-- !query 25 schema
|
||||||
|
struct<>
|
||||||
|
-- !query 25 output
|
||||||
|
|
||||||
|
|
|
@ -222,7 +222,10 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
|
||||||
val df = session.sql(sql)
|
val df = session.sql(sql)
|
||||||
val schema = df.schema
|
val schema = df.schema
|
||||||
// Get answer, but also get rid of the #1234 expression ids that show up in explain plans
|
// Get answer, but also get rid of the #1234 expression ids that show up in explain plans
|
||||||
val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x"))
|
val answer = df.queryExecution.hiveResultString().map(_.replaceAll("#\\d+", "#x")
|
||||||
|
.replaceAll("Location: .*/sql/core/", "Location: sql/core/")
|
||||||
|
.replaceAll("Created: .*\n", "Created: \n")
|
||||||
|
.replaceAll("Last Access: .*\n", "Last Access: \n"))
|
||||||
|
|
||||||
// If the output is not pre-sorted, sort it.
|
// If the output is not pre-sorted, sort it.
|
||||||
if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted)
|
if (isSorted(df.queryExecution.analyzed)) (schema, answer) else (schema, answer.sorted)
|
||||||
|
|
|
@ -977,40 +977,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
|
||||||
testRenamePartitions(isDatasourceTable = false)
|
testRenamePartitions(isDatasourceTable = false)
|
||||||
}
|
}
|
||||||
|
|
||||||
test("show table extended") {
|
|
||||||
withTempView("show1a", "show2b") {
|
|
||||||
sql(
|
|
||||||
"""
|
|
||||||
|CREATE TEMPORARY VIEW show1a
|
|
||||||
|USING org.apache.spark.sql.sources.DDLScanSource
|
|
||||||
|OPTIONS (
|
|
||||||
| From '1',
|
|
||||||
| To '10',
|
|
||||||
| Table 'test1'
|
|
||||||
|
|
|
||||||
|)
|
|
||||||
""".stripMargin)
|
|
||||||
sql(
|
|
||||||
"""
|
|
||||||
|CREATE TEMPORARY VIEW show2b
|
|
||||||
|USING org.apache.spark.sql.sources.DDLScanSource
|
|
||||||
|OPTIONS (
|
|
||||||
| From '1',
|
|
||||||
| To '10',
|
|
||||||
| Table 'test1'
|
|
||||||
|)
|
|
||||||
""".stripMargin)
|
|
||||||
assert(
|
|
||||||
sql("SHOW TABLE EXTENDED LIKE 'show*'").count() >= 2)
|
|
||||||
assert(
|
|
||||||
sql("SHOW TABLE EXTENDED LIKE 'show*'").schema ==
|
|
||||||
StructType(StructField("database", StringType, false) ::
|
|
||||||
StructField("tableName", StringType, false) ::
|
|
||||||
StructField("isTemporary", BooleanType, false) ::
|
|
||||||
StructField("information", StringType, false) :: Nil))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
test("show databases") {
|
test("show databases") {
|
||||||
sql("CREATE DATABASE showdb2B")
|
sql("CREATE DATABASE showdb2B")
|
||||||
sql("CREATE DATABASE showdb1A")
|
sql("CREATE DATABASE showdb1A")
|
||||||
|
|
Loading…
Reference in a new issue