From 4e5d2e06950ab60d2442a43e0656d47aca4a87f7 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Mon, 16 Nov 2020 07:08:21 +0000 Subject: [PATCH] [SPARK-33394][SQL][TESTS] Throw `NoSuchNamespaceException` for not existing namespace in `InMemoryTableCatalog.listTables()` ### What changes were proposed in this pull request? Throw `NoSuchNamespaceException` in `listTables()` of the custom test catalog `InMemoryTableCatalog` if the passed namespace doesn't exist. ### Why are the changes needed? 1. To align behavior of V2 `InMemoryTableCatalog` to V1 session catalog. 2. To distinguish two situations: 1. A namespace **does exist** but does not contain any tables. In that case, `listTables()` returns empty result. 2. A namespace **does not exist**. `listTables()` throws `NoSuchNamespaceException` in this case. ### Does this PR introduce _any_ user-facing change? Yes. For example, `SHOW TABLES` returns empty result before the changes. ### How was this patch tested? By running V1/V2 ShowTablesSuites. Closes #30358 from MaxGekk/show-tables-in-not-existing-namespace. Authored-by: Max Gekk Signed-off-by: Wenchen Fan --- .../spark/sql/connector/InMemoryTableCatalog.scala | 14 +++++++++++++- .../sql/connector/catalog/TableCatalogSuite.scala | 6 +++--- .../execution/command/ShowTablesSuiteBase.scala | 8 ++++++++ .../sql/execution/command/v1/ShowTablesSuite.scala | 9 --------- .../sql/execution/command/v2/ShowTablesSuite.scala | 6 ------ 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala index 6824efd988..edb9f65480 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTableCatalog.scala @@ -181,9 +181,21 @@ class InMemoryTableCatalog extends BasicInMemoryTableCatalog with SupportsNamesp override def dropNamespace(namespace: Array[String]): Boolean = { listNamespaces(namespace).foreach(dropNamespace) - listTables(namespace).foreach(dropTable) + try { + listTables(namespace).foreach(dropTable) + } catch { + case _: NoSuchNamespaceException => + } Option(namespaces.remove(namespace.toList)).isDefined } + + override def listTables(namespace: Array[String]): Array[Identifier] = { + if (namespace.isEmpty || namespaceExists(namespace)) { + super.listTables(namespace) + } else { + throw new NoSuchNamespaceException(namespace) + } + } } object InMemoryTableCatalog { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala index a4c85ec64e..dab20911bb 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/TableCatalogSuite.scala @@ -64,12 +64,12 @@ class TableCatalogSuite extends SparkFunSuite { val ident2 = Identifier.of(Array("ns"), "test_table_2") val ident3 = Identifier.of(Array("ns2"), "test_table_1") - assert(catalog.listTables(Array("ns")).isEmpty) + intercept[NoSuchNamespaceException](catalog.listTables(Array("ns"))) catalog.createTable(ident1, schema, Array.empty, emptyProps) assert(catalog.listTables(Array("ns")).toSet == Set(ident1)) - assert(catalog.listTables(Array("ns2")).isEmpty) + intercept[NoSuchNamespaceException](catalog.listTables(Array("ns2"))) catalog.createTable(ident3, schema, Array.empty, emptyProps) catalog.createTable(ident2, schema, Array.empty, emptyProps) @@ -841,7 +841,7 @@ class TableCatalogSuite extends SparkFunSuite { assert(catalog.dropNamespace(testNs)) assert(!catalog.namespaceExists(testNs)) - assert(catalog.listTables(testNs).isEmpty) + intercept[NoSuchNamespaceException](catalog.listTables(testNs)) } test("alterNamespace: basic behavior") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index 49428fab79..d7659e25d2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -21,6 +21,7 @@ import org.scalactic.source.Position import org.scalatest.Tag import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils @@ -57,6 +58,13 @@ trait ShowTablesSuiteBase extends QueryTest with SQLTestUtils { } } + test("show table in a not existing namespace") { + val msg = intercept[NoSuchNamespaceException] { + runShowTablesSql(s"SHOW TABLES IN $catalog.unknown", Seq()) + }.getMessage + assert(msg.matches("(Database|Namespace) 'unknown' not found;")) + } + test("show tables with a pattern") { withNamespace(s"$catalog.ns1", s"$catalog.ns2") { sql(s"CREATE NAMESPACE $catalog.ns1") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala index d2332818d9..5bbc6c6285 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.command.v1 import org.apache.spark.sql.{AnalysisException, Row} -import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.execution.command import org.apache.spark.sql.test.SharedSparkSession @@ -51,14 +50,6 @@ trait ShowTablesSuiteBase extends command.ShowTablesSuiteBase { } } - // `SHOW TABLES` returns empty result in V2 catalog instead of throwing the exception. - test("show table in a not existing namespace") { - val msg = intercept[NoSuchDatabaseException] { - runShowTablesSql(s"SHOW TABLES IN $catalog.unknown", Seq()) - }.getMessage - assert(msg.contains("Database 'unknown' not found")) - } - // `SHOW TABLES` from v2 catalog returns empty result. test("v1 SHOW TABLES list the temp views") { withSourceViews { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala index c7f68863a1..aff1729a00 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala @@ -44,12 +44,6 @@ class ShowTablesSuite extends command.ShowTablesSuiteBase with SharedSparkSessio override def sparkConf: SparkConf = super.sparkConf .set(s"spark.sql.catalog.$catalog", classOf[InMemoryTableCatalog].getName) - // The test fails with the exception `NoSuchDatabaseException` in V1 catalog. - // TODO(SPARK-33394): Throw `NoSuchDatabaseException` for not existing namespace - test("show table in a not existing namespace") { - runShowTablesSql(s"SHOW TABLES IN $catalog.unknown", Seq()) - } - // The test fails for V1 catalog with the error: // org.apache.spark.sql.AnalysisException: // The namespace in session catalog must have exactly one name part: spark_catalog.n1.n2.db