ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Remove extranious type definitions from inside of tests

Browse source
This commit is contained in:
Holden Karau 2013-10-21 00:20:15 -07:00
parent 699f7d28c0
commit 092b87e7c8
1 changed files with 86 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

172
core/src/test/scala/org/apache/spark/rdd/DoubleRDDSuite.scala
View file

@ -32,154 +32,154 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
test("WorksOnEmpty") { test("WorksOnEmpty") {
// Make sure that it works on an empty input // Make sure that it works on an empty input
val rdd: RDD[Double] = sc.parallelize(Seq()) val rdd: RDD[Double] = sc.parallelize(Seq())
val buckets: Array[Double] = Array(0.0, 10.0) val buckets = Array(0.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(0) val expectedHistogramResults = Array(0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksWithOutOfRangeWithOneBucket") { test("WorksWithOutOfRangeWithOneBucket") {
// Verify that if all of the elements are out of range the counts are zero // Verify that if all of the elements are out of range the counts are zero
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01)) val rdd = sc.parallelize(Seq(10.01, -0.01))
val buckets: Array[Double] = Array(0.0, 10.0) val buckets = Array(0.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(0) val expectedHistogramResults = Array(0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksInRangeWithOneBucket") { test("WorksInRangeWithOneBucket") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4)) val rdd = sc.parallelize(Seq(1, 2, 3, 4))
val buckets: Array[Double] = Array(0.0, 10.0) val buckets = Array(0.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(4) val expectedHistogramResults = Array(4)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksInRangeWithOneBucketExactMatch") { test("WorksInRangeWithOneBucketExactMatch") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4)) val rdd = sc.parallelize(Seq(1, 2, 3, 4))
val buckets: Array[Double] = Array(1.0, 4.0) val buckets = Array(1.0, 4.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(4) val expectedHistogramResults = Array(4)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksWithOutOfRangeWithTwoBuckets") { test("WorksWithOutOfRangeWithTwoBuckets") {
// Verify that out of range works with two buckets // Verify that out of range works with two buckets
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01)) val rdd = sc.parallelize(Seq(10.01, -0.01))
val buckets: Array[Double] = Array(0.0, 5.0, 10.0) val buckets = Array(0.0, 5.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(0, 0) val expectedHistogramResults = Array(0, 0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksWithOutOfRangeWithTwoUnEvenBuckets") { test("WorksWithOutOfRangeWithTwoUnEvenBuckets") {
// Verify that out of range works with two un even buckets // Verify that out of range works with two un even buckets
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01)) val rdd = sc.parallelize(Seq(10.01, -0.01))
val buckets: Array[Double] = Array(0.0, 4.0, 10.0) val buckets = Array(0.0, 4.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(0, 0) val expectedHistogramResults = Array(0, 0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
test("WorksInRangeWithTwoBuckets") { test("WorksInRangeWithTwoBuckets") {
// Make sure that it works with two equally spaced buckets and elements in each // Make sure that it works with two equally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6)) val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6))
val buckets: Array[Double] = Array(0.0, 5.0, 10.0) val buckets = Array(0.0, 5.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(3, 2) val expectedHistogramResults = Array(3, 2)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksInRangeWithTwoBucketsAndNaN") { test("WorksInRangeWithTwoBucketsAndNaN") {
// Make sure that it works with two equally spaced buckets and elements in each // Make sure that it works with two equally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6, Double.NaN)) val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6, Double.NaN))
val buckets: Array[Double] = Array(0.0, 5.0, 10.0) val buckets = Array(0.0, 5.0, 10.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val histogramResults2: Array[Long] = rdd.histogram(buckets, true) val histogramResults2 = rdd.histogram(buckets, true)
val expectedHistogramResults: Array[Long] = Array(3, 2) val expectedHistogramResults = Array(3, 2)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramResults2 === expectedHistogramResults) assert(histogramResults2 === expectedHistogramResults)
} }
test("WorksInRangeWithTwoUnevenBuckets") { test("WorksInRangeWithTwoUnevenBuckets") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6)) val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0) val buckets = Array(0.0, 5.0, 11.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(3, 2) val expectedHistogramResults = Array(3, 2)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
test("WorksMixedRangeWithTwoUnevenBuckets") { test("WorksMixedRangeWithTwoUnevenBuckets") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01)) val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0) val buckets = Array(0.0, 5.0, 11.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(4, 3) val expectedHistogramResults = Array(4, 3)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
test("WorksMixedRangeWithFourUnevenBuckets") { test("WorksMixedRangeWithFourUnevenBuckets") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
200.0, 200.1)) 200.0, 200.1))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0) val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 3) val expectedHistogramResults = Array(4, 2, 1, 3)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
test("WorksMixedRangeWithUnevenBucketsAndNaN") { test("WorksMixedRangeWithUnevenBucketsAndNaN") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
200.0, 200.1, Double.NaN)) 200.0, 200.1, Double.NaN))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0) val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 3) val expectedHistogramResults = Array(4, 2, 1, 3)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
// Make sure this works with a NaN end bucket // Make sure this works with a NaN end bucket
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRange") { test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRange") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
200.0, 200.1, Double.NaN)) 200.0, 200.1, Double.NaN))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN) val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 2, 3) val expectedHistogramResults = Array(4, 2, 1, 2, 3)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
// Make sure this works with a NaN end bucket and an inifity // Make sure this works with a NaN end bucket and an inifity
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRangeAndInfity") { test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRangeAndInfity") {
// Make sure that it works with two unequally spaced buckets and elements in each // Make sure that it works with two unequally spaced buckets and elements in each
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
200.0, 200.1, 1.0/0.0, -1.0/0.0, Double.NaN)) 200.0, 200.1, 1.0/0.0, -1.0/0.0, Double.NaN))
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN) val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 2, 4) val expectedHistogramResults = Array(4, 2, 1, 2, 4)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
test("WorksWithOutOfRangeWithInfiniteBuckets") { test("WorksWithOutOfRangeWithInfiniteBuckets") {
// Verify that out of range works with two buckets // Verify that out of range works with two buckets
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01, Double.NaN)) val rdd = sc.parallelize(Seq(10.01, -0.01, Double.NaN))
val buckets: Array[Double] = Array(-1.0/0.0 , 0.0, 1.0/0.0) val buckets = Array(-1.0/0.0 , 0.0, 1.0/0.0)
val histogramResults: Array[Long] = rdd.histogram(buckets) val histogramResults = rdd.histogram(buckets)
val expectedHistogramResults: Array[Long] = Array(1, 1) val expectedHistogramResults = Array(1, 1)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
} }
// Test the failure mode with an invalid bucket array // Test the failure mode with an invalid bucket array
test("ThrowsExceptionOnInvalidBucketArray") { test("ThrowsExceptionOnInvalidBucketArray") {
val rdd: RDD[Double] = sc.parallelize(Seq(1.0)) val rdd = sc.parallelize(Seq(1.0))
// Empty array // Empty array
intercept[IllegalArgumentException] { intercept[IllegalArgumentException] {
val buckets: Array[Double] = Array.empty[Double] val buckets = Array.empty[Double]
val result = rdd.histogram(buckets) val result = rdd.histogram(buckets)
} }
// Single element array // Single element array
intercept[IllegalArgumentException] { intercept[IllegalArgumentException] {
val buckets: Array[Double] = Array(1.0) val buckets = Array(1.0)
val result = rdd.histogram(buckets) val result = rdd.histogram(buckets)
} }
} }
@ -187,49 +187,49 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
// Test automatic histogram function // Test automatic histogram function
test("WorksWithoutBucketsBasic") { test("WorksWithoutBucketsBasic") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4)) val rdd = sc.parallelize(Seq(1, 2, 3, 4))
val (histogramBuckets, histogramResults) = rdd.histogram(1) val (histogramBuckets, histogramResults) = rdd.histogram(1)
val expectedHistogramResults: Array[Long] = Array(4) val expectedHistogramResults = Array(4)
val expectedHistogramBuckets: Array[Double] = Array(1.0, 4.0) val expectedHistogramBuckets = Array(1.0, 4.0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramBuckets === expectedHistogramBuckets) assert(histogramBuckets === expectedHistogramBuckets)
} }
// Test automatic histogram function with a single element // Test automatic histogram function with a single element
test("WorksWithoutBucketsBasicSingleElement") { test("WorksWithoutBucketsBasicSingleElement") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1)) val rdd = sc.parallelize(Seq(1))
val (histogramBuckets, histogramResults) = rdd.histogram(1) val (histogramBuckets, histogramResults) = rdd.histogram(1)
val expectedHistogramResults: Array[Long] = Array(1) val expectedHistogramResults = Array(1)
val expectedHistogramBuckets: Array[Double] = Array(1.0, 1.0) val expectedHistogramBuckets = Array(1.0, 1.0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramBuckets === expectedHistogramBuckets) assert(histogramBuckets === expectedHistogramBuckets)
} }
// Test automatic histogram function with a single element // Test automatic histogram function with a single element
test("WorksWithoutBucketsBasicNoRange") { test("WorksWithoutBucketsBasicNoRange") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 1, 1, 1)) val rdd = sc.parallelize(Seq(1, 1, 1, 1))
val (histogramBuckets, histogramResults) = rdd.histogram(1) val (histogramBuckets, histogramResults) = rdd.histogram(1)
val expectedHistogramResults: Array[Long] = Array(4) val expectedHistogramResults = Array(4)
val expectedHistogramBuckets: Array[Double] = Array(1.0, 1.0) val expectedHistogramBuckets = Array(1.0, 1.0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramBuckets === expectedHistogramBuckets) assert(histogramBuckets === expectedHistogramBuckets)
} }
test("WorksWithoutBucketsBasicTwo") { test("WorksWithoutBucketsBasicTwo") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4)) val rdd = sc.parallelize(Seq(1, 2, 3, 4))
val (histogramBuckets, histogramResults) = rdd.histogram(2) val (histogramBuckets, histogramResults) = rdd.histogram(2)
val expectedHistogramResults: Array[Long] = Array(2, 2) val expectedHistogramResults = Array(2, 2)
val expectedHistogramBuckets: Array[Double] = Array(1.0, 2.5, 4.0) val expectedHistogramBuckets = Array(1.0, 2.5, 4.0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramBuckets === expectedHistogramBuckets) assert(histogramBuckets === expectedHistogramBuckets)
} }
test("WorksWithoutBucketsWithMoreRequestedThanElements") { test("WorksWithoutBucketsWithMoreRequestedThanElements") {
// Verify the basic case of one bucket and all elements in that bucket works // Verify the basic case of one bucket and all elements in that bucket works
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2)) val rdd = sc.parallelize(Seq(1, 2))
val (histogramBuckets, histogramResults) = rdd.histogram(10) val (histogramBuckets, histogramResults) = rdd.histogram(10)
val expectedHistogramResults: Array[Long] = val expectedHistogramResults =
Array(1, 0, 0, 0, 0, 0, 0, 0, 0, 1) Array(1, 0, 0, 0, 0, 0, 0, 0, 0, 1)
val expectedHistogramBuckets: Array[Double] = val expectedHistogramBuckets =
Array(1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0) Array(1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0)
assert(histogramResults === expectedHistogramResults) assert(histogramResults === expectedHistogramResults)
assert(histogramBuckets === expectedHistogramBuckets) assert(histogramBuckets === expectedHistogramBuckets)
@ -239,12 +239,12 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
test("ThrowsExceptionOnInvalidRDDs") { test("ThrowsExceptionOnInvalidRDDs") {
// infinity // infinity
intercept[UnsupportedOperationException] { intercept[UnsupportedOperationException] {
val rdd: RDD[Double] = sc.parallelize(Seq(1, 1.0/0.0)) val rdd = sc.parallelize(Seq(1, 1.0/0.0))
val result = rdd.histogram(1) val result = rdd.histogram(1)
} }
// NaN // NaN
intercept[UnsupportedOperationException] { intercept[UnsupportedOperationException] {
val rdd: RDD[Double] = sc.parallelize(Seq(1, Double.NaN)) val rdd = sc.parallelize(Seq(1, Double.NaN))
val result = rdd.histogram(1) val result = rdd.histogram(1)
} }
// Empty // Empty