ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix for partitioning when sorting in descending order

Browse source
This commit is contained in:
Harvey 2012-08-03 12:22:07 -07:00
parent 4a9c58913d
commit 5ec13327d4
1 changed files with 1 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
core/src/main/scala/spark/Partitioner.scala
View file

@ -39,8 +39,7 @@ class RangePartitioner[K <% Ordered[K]: ClassManifest, V](
val rddSize = rdd.count()
val maxSampleSize = partitions * 10.0
val frac = math.min(maxSampleSize / math.max(rddSize, 1), 1.0)
val rddSample = rdd.sample(true, frac, 1).map(_._1).collect()
.sortWith((x, y) => if (ascending) x < y else x > y)
val rddSample = rdd.sample(true, frac, 1).map(_._1).collect().sortWith(_ < _)
if (rddSample.length == 0) {
Array()
} else {