ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35687][SQL][TEST] PythonUDFSuite move assume into its methods

Browse source 
### What changes were proposed in this pull request?

Move `assume` into methods at `PythonUDFSuite`.

### Why are the changes needed?

When we run Spark test with such command:
`./build/mvn -Phadoop-2.7 -Phive -Phive-thriftserver -Pyarn -Pkubernetes clean test`

get this exception:
```
 PythonUDFSuite:
 org.apache.spark.sql.execution.python.PythonUDFSuite *** ABORTED ***
   java.lang.RuntimeException: Unable to load a Suite class that was discovered in the runpath: org.apache.spark.sql.execution.python.PythonUDFSuite
   at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:81)
   at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38)
   at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
   at scala.collection.Iterator.foreach(Iterator.scala:941)
   at scala.collection.Iterator.foreach$(Iterator.scala:941)
   at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
   at scala.collection.IterableLike.foreach(IterableLike.scala:74)
   at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
   at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
   at scala.collection.TraversableLike.map(TraversableLike.scala:238)
```

The test env has no PYSpark module so it failed.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

manual

Closes #32833 from ulysses-you/SPARK-35687.

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
This commit is contained in:
ulysses-you 2021-06-09 15:57:56 +09:00 committed by Hyukjin Kwon
parent 9f010a8eb2
commit 825b620862
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala
View file

@ -28,7 +28,6 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession {
val scalaTestUDF = TestScalaUDF(name = "scalaUDF")
val pythonTestUDF = TestPythonUDF(name = "pyUDF")
assume(shouldTestPythonUDFs)
lazy val base = Seq(
(Some(1), Some(1)), (Some(1), Some(2)), (Some(2), Some(1)),
@ -36,6 +35,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession {
(None, Some(1)), (Some(3), None), (None, None)).toDF("a", "b")
test("SPARK-28445: PythonUDF as grouping key and aggregate expressions") {
assume(shouldTestPythonUDFs)
val df1 = base.groupBy(scalaTestUDF(base("a") + 1))
.agg(scalaTestUDF(base("a") + 1), scalaTestUDF(count(base("b"))))
val df2 = base.groupBy(pythonTestUDF(base("a") + 1))
@ -44,6 +44,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession {
}
test("SPARK-28445: PythonUDF as grouping key and used in aggregate expressions") {
assume(shouldTestPythonUDFs)
val df1 = base.groupBy(scalaTestUDF(base("a") + 1))
.agg(scalaTestUDF(base("a") + 1) + 1, scalaTestUDF(count(base("b"))))
val df2 = base.groupBy(pythonTestUDF(base("a") + 1))
@ -52,6 +53,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession {
}
test("SPARK-28445: PythonUDF in aggregate expression has grouping key in its arguments") {
assume(shouldTestPythonUDFs)
val df1 = base.groupBy(scalaTestUDF(base("a") + 1))
.agg(scalaTestUDF(scalaTestUDF(base("a") + 1)), scalaTestUDF(count(base("b"))))
val df2 = base.groupBy(pythonTestUDF(base("a") + 1))
@ -60,6 +62,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession {
}
test("SPARK-28445: PythonUDF over grouping key is argument to aggregate function") {
assume(shouldTestPythonUDFs)
val df1 = base.groupBy(scalaTestUDF(base("a") + 1))
.agg(scalaTestUDF(scalaTestUDF(base("a") + 1)),
scalaTestUDF(count(scalaTestUDF(base("a") + 1))))