ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-34325][CORE] Remove unused shuffleBlockResolver variable inSortShuffleWriter

Browse source 
### What changes were proposed in this pull request?
Remove unused shuffleBlockResolver variable in SortShuffleWriter.

### Why are the changes needed?
For better code understanding.

### Does this PR introduce _any_ user-facing change?
NO.

### How was this patch tested?
End to End.

Closes #31433 from offthewall123/remove_shuffleBlockResolver_in_SortShuffleWriter.

Authored-by: offthewall123 <dingyu.xu@intel.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
offthewall123 2021-02-03 09:40:08 +09:00 committed by HyukjinKwon
parent 00120ea537
commit 60c71c6d2d
3 changed files with 2 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
View file

@ -166,8 +166,7 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
metrics, metrics,
shuffleExecutorComponents) shuffleExecutorComponents)
case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] => case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] =>
new SortShuffleWriter( new SortShuffleWriter(other, mapId, context, shuffleExecutorComponents)
shuffleBlockResolver, other, mapId, context, shuffleExecutorComponents)
} }
} }

3
core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
View file

@ -20,12 +20,11 @@ package org.apache.spark.shuffle.sort
import org.apache.spark._ import org.apache.spark._
import org.apache.spark.internal.{config, Logging} import org.apache.spark.internal.{config, Logging}
import org.apache.spark.scheduler.MapStatus import org.apache.spark.scheduler.MapStatus
import org.apache.spark.shuffle.{BaseShuffleHandle, IndexShuffleBlockResolver, ShuffleWriter} import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleWriter}
import org.apache.spark.shuffle.api.ShuffleExecutorComponents import org.apache.spark.shuffle.api.ShuffleExecutorComponents
import org.apache.spark.util.collection.ExternalSorter import org.apache.spark.util.collection.ExternalSorter
private[spark] class SortShuffleWriter[K, V, C]( private[spark] class SortShuffleWriter[K, V, C](
shuffleBlockResolver: IndexShuffleBlockResolver,
handle: BaseShuffleHandle[K, V, C], handle: BaseShuffleHandle[K, V, C],
mapId: Long, mapId: Long,
context: TaskContext, context: TaskContext,

2
core/src/test/scala/org/apache/spark/shuffle/sort/SortShuffleWriterSuite.scala
View file

@ -74,7 +74,6 @@ class SortShuffleWriterSuite extends SparkFunSuite with SharedSparkContext with
test("write empty iterator") { test("write empty iterator") {
val context = MemoryTestingUtils.fakeTaskContext(sc.env) val context = MemoryTestingUtils.fakeTaskContext(sc.env)
val writer = new SortShuffleWriter[Int, Int, Int]( val writer = new SortShuffleWriter[Int, Int, Int](
shuffleBlockResolver,
shuffleHandle, shuffleHandle,
mapId = 1, mapId = 1,
context, context,
@ -92,7 +91,6 @@ class SortShuffleWriterSuite extends SparkFunSuite with SharedSparkContext with
val context = MemoryTestingUtils.fakeTaskContext(sc.env) val context = MemoryTestingUtils.fakeTaskContext(sc.env)
val records = List[(Int, Int)]((1, 2), (2, 3), (4, 4), (6, 5)) val records = List[(Int, Int)]((1, 2), (2, 3), (4, 4), (6, 5))
val writer = new SortShuffleWriter[Int, Int, Int]( val writer = new SortShuffleWriter[Int, Int, Int](
shuffleBlockResolver,
shuffleHandle, shuffleHandle,
mapId = 2, mapId = 2,
context, context,