[SPARK-29442][SQL] Set default
mode should override the existing mode
### What changes were proposed in this pull request? This PR aims to fix the behavior of `mode("default")` to set `SaveMode.ErrorIfExists`. Also, this PR updates the exception message by adding `default` explicitly. ### Why are the changes needed? This is reported during `GRAPH API` PR. This builder pattern should work like the documentation. ### Does this PR introduce any user-facing change? Yes if the app has multiple `mode()` invocation including `mode("default")` and the `mode("default")` is the last invocation. This is really a corner case. - Previously, the last invocation was handled as `No-Op`. - After this bug fix, it will work like the documentation. ### How was this patch tested? Pass the Jenkins with the newly added test case. Closes #26094 from dongjoon-hyun/SPARK-29442. Authored-by: Dongjoon Hyun <dhyun@apple.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
parent
81915dacc4
commit
e696c36e32
|
@ -87,10 +87,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
|
|||
case "overwrite" => mode(SaveMode.Overwrite)
|
||||
case "append" => mode(SaveMode.Append)
|
||||
case "ignore" => mode(SaveMode.Ignore)
|
||||
case "error" | "errorifexists" => mode(SaveMode.ErrorIfExists)
|
||||
case "default" => this
|
||||
case _ => throw new IllegalArgumentException(s"Unknown save mode: $saveMode. " +
|
||||
"Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.")
|
||||
case "error" | "errorifexists" | "default" => mode(SaveMode.ErrorIfExists)
|
||||
case _ => throw new IllegalArgumentException(s"Unknown save mode: $saveMode. Accepted " +
|
||||
"save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists', 'default'.")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -2203,4 +2203,12 @@ class DataFrameSuite extends QueryTest with SharedSparkSession {
|
|||
|*(1) Range (0, 10, step=1, splits=2)""".stripMargin))
|
||||
}
|
||||
}
|
||||
|
||||
test("SPARK-29442 Set `default` mode should override the existing mode") {
|
||||
val df = Seq(Tuple1(1)).toDF()
|
||||
val writer = df.write.mode("overwrite").mode("default")
|
||||
val modeField = classOf[DataFrameWriter[Tuple1[Int]]].getDeclaredField("mode")
|
||||
modeField.setAccessible(true)
|
||||
assert(SaveMode.ErrorIfExists === modeField.get(writer).asInstanceOf[SaveMode])
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue