made preferredLocation a val of the surrounding case class

This commit is contained in:
Ali Ghodsi 2013-08-15 11:03:42 -07:00
parent f24861b60a
commit f67753cdfc

View file

@ -25,7 +25,7 @@ private[spark] case class CoalescedRDDPartition(
index: Int,
@transient rdd: RDD[_],
parentsIndices: Array[Int],
prefLoc: String = ""
@transient preferredLocation: String = ""
) extends Partition {
var parents: Seq[Partition] = parentsIndices.map(rdd.partitions(_))
@ -36,13 +36,6 @@ private[spark] case class CoalescedRDDPartition(
oos.defaultWriteObject()
}
/**
* Gets the preferred location for this coalesced RDD partition.
* Most parent indices should prefer this machine.
* @return preferred location
*/
def getPreferredLocation = prefLoc
/**
* Computes how many of the parents partitions have getPreferredLocation
* as one of their preferredLocations
@ -50,7 +43,7 @@ private[spark] case class CoalescedRDDPartition(
*/
def localFraction :Double = {
var loc: Int = 0
parents.foreach(p => if (rdd.preferredLocations(p).contains(getPreferredLocation)) loc += 1)
parents.foreach(p => if (rdd.preferredLocations(p).contains(preferredLocation)) loc += 1)
if (parents.size == 0) 0.0 else (loc.toDouble / parents.size.toDouble)
}
}
@ -128,7 +121,7 @@ class CoalescedRDD[T: ClassManifest](
*/
override def getPreferredLocations(partition: Partition): Seq[String] = {
if (partition.isInstanceOf[CoalescedRDDPartition])
List(partition.asInstanceOf[CoalescedRDDPartition].getPreferredLocation)
List(partition.asInstanceOf[CoalescedRDDPartition].preferredLocation)
else
super.getPreferredLocations(partition)
}