ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Updated PruneDependency to change "split" to "partition".

Browse source
This commit is contained in:
Reynold Xin 2013-01-23 22:22:03 -08:00
parent eedc542a02
commit c109f29c97
2 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
core/src/main/scala/spark/Dependency.scala
View file

@ -64,14 +64,14 @@ class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
/** /**
* Represents a dependency between the SplitsPruningRDD and its parent. In this * Represents a dependency between the PartitionPruningRDD and its parent. In this
* case, the child RDD contains a subset of splits of the parents'. * case, the child RDD contains a subset of partitions of the parents'.
*/ */
class PruneDependency[T](rdd: RDD[T], @transient splitsFilterFunc: Int => Boolean) class PruneDependency[T](rdd: RDD[T], @transient partitionFilterFunc: Int => Boolean)
extends NarrowDependency[T](rdd) { extends NarrowDependency[T](rdd) {
@transient @transient
val splits: Array[Split] = rdd.splits.filter(s => splitsFilterFunc(s.index)) val partitions: Array[Split] = rdd.splits.filter(s => partitionFilterFunc(s.index))
override def getParents(partitionId: Int) = List(splits(partitionId).index) override def getParents(partitionId: Int) = List(partitions(partitionId).index)
} }

2
core/src/main/scala/spark/rdd/PartitionPruningRDD.scala
View file

@ -14,7 +14,7 @@ class PartitionPruningRDD[T: ClassManifest](
extends RDD[T](prev.context, List(new PruneDependency(prev, partitionFilterFunc))) { extends RDD[T](prev.context, List(new PruneDependency(prev, partitionFilterFunc))) {
@transient @transient
val partitions_ : Array[Split] = dependencies_.head.asInstanceOf[PruneDependency[T]].splits val partitions_ : Array[Split] = dependencies_.head.asInstanceOf[PruneDependency[T]].partitions
override def compute(split: Split, context: TaskContext) = firstParent[T].iterator(split, context) override def compute(split: Split, context: TaskContext) = firstParent[T].iterator(split, context)