ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-36381][SQL] Add case sensitive and case insensitive compare for checking column name exist when alter table

Browse source 
### What changes were proposed in this pull request?
Add the Resolver to `checkColumnNotExists` to check name exist in case sensitive.

### Why are the changes needed?
At now the resolver is `_ == _` of `findNestedField`  called by `checkColumnNotExists`
Add `alter.conf.resolver` to it.
[SPARK-36381](https://issues.apache.org/jira/browse/SPARK-36381)
### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Add ut tests

Closes #33618 from Peng-Lei/sensitive-cloumn-name.

Authored-by: PengLei <peng.8lei@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
This commit is contained in:
PengLei 2021-08-04 10:04:13 +09:00 committed by Hyukjin Kwon
parent 4a6afb4875
commit 87d49cbcb1
2 changed files with 27 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
View file

@ -940,8 +940,9 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog {
* Validates the options used for alter table commands after table and columns are resolved.
*/
private def checkAlterTableCommand(alter: AlterTableCommand): Unit = {
def checkColumnNotExists(op: String, fieldNames: Seq[String], struct: StructType): Unit = {
if (struct.findNestedField(fieldNames, includeCollections = true).isDefined) {
def checkColumnNotExists(
op: String, fieldNames: Seq[String], struct: StructType, r: Resolver): Unit = {
if (struct.findNestedField(fieldNames, includeCollections = true, r).isDefined) {
alter.failAnalysis(s"Cannot $op column, because ${fieldNames.quoted} " +
s"already exists in ${struct.treeString}")
}
@ -950,7 +951,7 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog {
alter match {
case AddColumns(table: ResolvedTable, colsToAdd) =>
colsToAdd.foreach { colToAdd =>
checkColumnNotExists("add", colToAdd.name, table.schema)
checkColumnNotExists("add", colToAdd.name, table.schema, alter.conf.resolver)
}
SchemaUtils.checkColumnNameDuplication(
colsToAdd.map(_.name.quoted),
@ -958,7 +959,7 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog {
alter.conf.resolver)
case RenameColumn(table: ResolvedTable, col: ResolvedFieldName, newName) =>
checkColumnNotExists("rename", col.path :+ newName, table.schema)
checkColumnNotExists("rename", col.path :+ newName, table.schema, alter.conf.resolver)
case a @ AlterColumn(table: ResolvedTable, col: ResolvedFieldName, _, _, _, _) =>
val fieldName = col.name.quoted

22
sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
View file

@ -249,6 +249,28 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
expectErrorOnCaseSensitive = false)
}
test("SPARK-36381: Check column name exist case sensitive and insensitive when add column") {
alterTableTest(
AddColumns(
table,
Seq(QualifiedColType(
None,
"ID",
LongType,
true,
None,
Some(UnresolvedFieldPosition(ColumnPosition.after("id")))))),
Seq("Cannot add column, because ID already exists in root"),
expectErrorOnCaseSensitive = false)
}
test("SPARK-36381: Check column name exist case sensitive and insensitive when rename column") {
alterTableTest(
RenameColumn(table, UnresolvedFieldName(Array("id")), "DATA"),
Seq("Cannot rename column, because DATA already exists in root"),
expectErrorOnCaseSensitive = false)
}
test("AlterTable: drop column resolution") {
Seq(Array("ID"), Array("point", "X"), Array("POINT", "X"), Array("POINT", "x")).foreach { ref =>
alterTableTest(