[SPARK-15929] Fix portability of DataFrameSuite path globbing tests
The DataFrameSuite regression tests for SPARK-13774 fail in my environment because they attempt to glob over all of `/mnt` and some of the subdirectories restrictive permissions which cause the test to fail. This patch rewrites those tests to remove all environment-specific assumptions; the tests now create their own unique temporary paths for use in the tests. Author: Josh Rosen <joshrosen@databricks.com> Closes #13649 from JoshRosen/SPARK-15929.
This commit is contained in:
parent
ced8d669b3
commit
a6babca1bf
|
@ -19,6 +19,7 @@ package org.apache.spark.sql
|
|||
|
||||
import java.io.File
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.UUID
|
||||
|
||||
import scala.language.postfixOps
|
||||
import scala.util.Random
|
||||
|
@ -35,6 +36,7 @@ import org.apache.spark.sql.internal.SQLConf
|
|||
import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSQLContext}
|
||||
import org.apache.spark.sql.test.SQLTestData.TestData2
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.util.Utils
|
||||
|
||||
class DataFrameSuite extends QueryTest with SharedSQLContext {
|
||||
import testImplicits._
|
||||
|
@ -1495,18 +1497,43 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
|
|||
}
|
||||
|
||||
test("SPARK-13774: Check error message for non existent path without globbed paths") {
|
||||
val e = intercept[AnalysisException] (spark.read.format("csv").
|
||||
load("/xyz/file2", "/xyz/file21", "/abc/files555", "a")).getMessage()
|
||||
assert(e.startsWith("Path does not exist"))
|
||||
val uuid = UUID.randomUUID().toString
|
||||
val baseDir = Utils.createTempDir()
|
||||
try {
|
||||
val e = intercept[AnalysisException] {
|
||||
spark.read.format("csv").load(
|
||||
new File(baseDir, "file").getAbsolutePath,
|
||||
new File(baseDir, "file2").getAbsolutePath,
|
||||
new File(uuid, "file3").getAbsolutePath,
|
||||
uuid).rdd
|
||||
}
|
||||
assert(e.getMessage.startsWith("Path does not exist"))
|
||||
} finally {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
test("SPARK-13774: Check error message for not existent globbed paths") {
|
||||
val e = intercept[AnalysisException] (spark.read.format("text").
|
||||
load( "/xyz/*")).getMessage()
|
||||
assert(e.startsWith("Path does not exist"))
|
||||
// Non-existent initial path component:
|
||||
val nonExistentBasePath = "/" + UUID.randomUUID().toString
|
||||
assert(!new File(nonExistentBasePath).exists())
|
||||
val e = intercept[AnalysisException] {
|
||||
spark.read.format("text").load(s"$nonExistentBasePath/*")
|
||||
}
|
||||
assert(e.getMessage.startsWith("Path does not exist"))
|
||||
|
||||
val e1 = intercept[AnalysisException] (spark.read.json("/mnt/*/*-xyz.json").rdd).
|
||||
getMessage()
|
||||
assert(e1.startsWith("Path does not exist"))
|
||||
// Existent initial path component, but no matching files:
|
||||
val baseDir = Utils.createTempDir()
|
||||
val childDir = Utils.createTempDir(baseDir.getAbsolutePath)
|
||||
assert(childDir.exists())
|
||||
try {
|
||||
val e1 = intercept[AnalysisException] {
|
||||
spark.read.json(s"${baseDir.getAbsolutePath}/*/*-xyz.json").rdd
|
||||
}
|
||||
assert(e1.getMessage.startsWith("Path does not exist"))
|
||||
} finally {
|
||||
Utils.deleteRecursively(baseDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue