[SPARK-35639][SQL][FOLLOWUP] Make hasCoalescedPartition return true if something was actually coalesced
### What changes were proposed in this pull request?
Add `CoalescedPartitionSpec(0, 0, _)` check if a `CoalescedPartitionSpec` is coalesced.
### Why are the changes needed?
Fix corner case.
### Does this PR introduce _any_ user-facing change?
yes, UI may be changed
### How was this patch tested?
Add test
Closes #33342 from ulysses-you/SPARK-35639-FOLLOW.
Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3819641201
)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
This commit is contained in:
parent
4cc7d9b8f1
commit
0da71548a5
|
@ -92,6 +92,8 @@ case class CustomShuffleReaderExec private(
|
||||||
*/
|
*/
|
||||||
def hasCoalescedPartition: Boolean = {
|
def hasCoalescedPartition: Boolean = {
|
||||||
partitionSpecs.exists {
|
partitionSpecs.exists {
|
||||||
|
// shuffle from empty RDD
|
||||||
|
case CoalescedPartitionSpec(0, 0, _) => true
|
||||||
case s: CoalescedPartitionSpec => s.endReducerIndex - s.startReducerIndex > 1
|
case s: CoalescedPartitionSpec => s.endReducerIndex - s.startReducerIndex > 1
|
||||||
case _ => false
|
case _ => false
|
||||||
}
|
}
|
||||||
|
|
|
@ -1692,7 +1692,9 @@ class AdaptiveQueryExecSuite
|
||||||
val (_, adaptive) = runAdaptiveAndVerifyResult("SELECT c1, count(*) FROM t GROUP BY c1")
|
val (_, adaptive) = runAdaptiveAndVerifyResult("SELECT c1, count(*) FROM t GROUP BY c1")
|
||||||
assert(
|
assert(
|
||||||
collect(adaptive) {
|
collect(adaptive) {
|
||||||
case c @ CustomShuffleReaderExec(_, partitionSpecs) if partitionSpecs.length == 1 => c
|
case c @ CustomShuffleReaderExec(_, partitionSpecs) if partitionSpecs.length == 1 =>
|
||||||
|
assert(c.hasCoalescedPartition)
|
||||||
|
c
|
||||||
}.length == 1
|
}.length == 1
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue