Merge pull request #536 from sasurfer/master
CoalescedRDD for many partitions
This commit is contained in:
commit
945d1e720e
|
@ -37,8 +37,8 @@ class CoalescedRDD[T: ClassManifest](
|
||||||
prevSplits.map(_.index).map{idx => new CoalescedRDDPartition(idx, prev, Array(idx)) }
|
prevSplits.map(_.index).map{idx => new CoalescedRDDPartition(idx, prev, Array(idx)) }
|
||||||
} else {
|
} else {
|
||||||
(0 until maxPartitions).map { i =>
|
(0 until maxPartitions).map { i =>
|
||||||
val rangeStart = (i * prevSplits.length) / maxPartitions
|
val rangeStart = ((i.toLong * prevSplits.length) / maxPartitions).toInt
|
||||||
val rangeEnd = ((i + 1) * prevSplits.length) / maxPartitions
|
val rangeEnd = (((i.toLong + 1) * prevSplits.length) / maxPartitions).toInt
|
||||||
new CoalescedRDDPartition(i, prev, (rangeStart until rangeEnd).toArray)
|
new CoalescedRDDPartition(i, prev, (rangeStart until rangeEnd).toArray)
|
||||||
}.toArray
|
}.toArray
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue