spark-instrumented-optimizer/core/src/main/scala/spark/Dependency.scala

31 lines
856 B
Scala
Raw Normal View History

2011-02-27 22:15:52 -05:00
package spark
@serializable
abstract class Dependency[T](val rdd: RDD[T], val isShuffle: Boolean)
abstract class NarrowDependency[T](rdd: RDD[T])
extends Dependency(rdd, false) {
def getParents(outputPartition: Int): Seq[Int]
}
class ShuffleDependency[K, V, C](
val shuffleId: Int,
rdd: RDD[(K, V)],
val aggregator: Aggregator[K, V, C],
2011-03-07 02:38:16 -05:00
val partitioner: Partitioner
2011-02-27 22:15:52 -05:00
) extends Dependency(rdd, true)
class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
override def getParents(partitionId: Int) = List(partitionId)
}
class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
extends NarrowDependency[T](rdd) {
override def getParents(partitionId: Int) = {
if (partitionId >= outStart && partitionId < outStart + length)
List(partitionId - outStart + inStart)
else
Nil
}
}