ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' into mesos-0.9

Browse source
This commit is contained in:
Matei Zaharia 2012-04-05 14:53:22 -07:00
parent acaf99cb3a 816d4e5840
commit a8bb324ed9
2 changed files with 31 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
core/src/main/scala/spark/RDD.scala
View file

@ -94,37 +94,37 @@ abstract class RDD[T: ClassManifest](@transient sc: SparkContext) extends Serial
new SampledRDD(this, withReplacement, fraction, seed) new SampledRDD(this, withReplacement, fraction, seed)
def takeSample(withReplacement: Boolean, num: Int, seed: Int): Array[T] = { def takeSample(withReplacement: Boolean, num: Int, seed: Int): Array[T] = {
var fraction = 0.0 var fraction = 0.0
var total = 0 var total = 0
var multiplier = 3.0 var multiplier = 3.0
var initialCount = count() var initialCount = count()
var maxSelected = 0 var maxSelected = 0
if (initialCount > Integer.MAX_VALUE) { if (initialCount > Integer.MAX_VALUE) {
maxSelected = Integer.MAX_VALUE maxSelected = Integer.MAX_VALUE
} else { } else {
maxSelected = initialCount.toInt maxSelected = initialCount.toInt
} }
if (num > initialCount) { if (num > initialCount) {
total = maxSelected total = maxSelected
fraction = Math.min(multiplier * (maxSelected + 1) / initialCount, 1.0) fraction = Math.min(multiplier * (maxSelected + 1) / initialCount, 1.0)
} else if (num < 0) { } else if (num < 0) {
throw(new IllegalArgumentException("Negative number of elements requested")) throw(new IllegalArgumentException("Negative number of elements requested"))
} else { } else {
fraction = Math.min(multiplier * (num + 1) / initialCount, 1.0) fraction = Math.min(multiplier * (num + 1) / initialCount, 1.0)
total = num.toInt total = num.toInt
} }
var samples = this.sample(withReplacement, fraction, seed).collect() var samples = this.sample(withReplacement, fraction, seed).collect()
while (samples.length < total) { while (samples.length < total) {
samples = this.sample(withReplacement, fraction, seed).collect() samples = this.sample(withReplacement, fraction, seed).collect()
} }
val arr = samples.take(total) val arr = samples.take(total)
return arr return arr
} }
def union(other: RDD[T]): RDD[T] = new UnionRDD(sc, Array(this, other)) def union(other: RDD[T]): RDD[T] = new UnionRDD(sc, Array(this, other))

2
core/src/main/scala/spark/SparkContext.scala
View file

@ -44,7 +44,7 @@ class SparkContext(
// Set Spark master host and port system properties // Set Spark master host and port system properties
if (System.getProperty("spark.master.host") == null) { if (System.getProperty("spark.master.host") == null) {
System.setProperty("spark.master.host", Utils.localHostName) System.setProperty("spark.master.host", Utils.localIpAddress)
} }
if (System.getProperty("spark.master.port") == null) { if (System.getProperty("spark.master.port") == null) {
System.setProperty("spark.master.port", "7077") System.setProperty("spark.master.port", "7077")