Remove extranious type definitions from inside of tests
This commit is contained in:
parent
699f7d28c0
commit
092b87e7c8
|
@ -32,154 +32,154 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
|
||||||
test("WorksOnEmpty") {
|
test("WorksOnEmpty") {
|
||||||
// Make sure that it works on an empty input
|
// Make sure that it works on an empty input
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq())
|
val rdd: RDD[Double] = sc.parallelize(Seq())
|
||||||
val buckets: Array[Double] = Array(0.0, 10.0)
|
val buckets = Array(0.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(0)
|
val expectedHistogramResults = Array(0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksWithOutOfRangeWithOneBucket") {
|
test("WorksWithOutOfRangeWithOneBucket") {
|
||||||
// Verify that if all of the elements are out of range the counts are zero
|
// Verify that if all of the elements are out of range the counts are zero
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01))
|
val rdd = sc.parallelize(Seq(10.01, -0.01))
|
||||||
val buckets: Array[Double] = Array(0.0, 10.0)
|
val buckets = Array(0.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(0)
|
val expectedHistogramResults = Array(0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksInRangeWithOneBucket") {
|
test("WorksInRangeWithOneBucket") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 4))
|
||||||
val buckets: Array[Double] = Array(0.0, 10.0)
|
val buckets = Array(0.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4)
|
val expectedHistogramResults = Array(4)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksInRangeWithOneBucketExactMatch") {
|
test("WorksInRangeWithOneBucketExactMatch") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 4))
|
||||||
val buckets: Array[Double] = Array(1.0, 4.0)
|
val buckets = Array(1.0, 4.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4)
|
val expectedHistogramResults = Array(4)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksWithOutOfRangeWithTwoBuckets") {
|
test("WorksWithOutOfRangeWithTwoBuckets") {
|
||||||
// Verify that out of range works with two buckets
|
// Verify that out of range works with two buckets
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01))
|
val rdd = sc.parallelize(Seq(10.01, -0.01))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 10.0)
|
val buckets = Array(0.0, 5.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(0, 0)
|
val expectedHistogramResults = Array(0, 0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksWithOutOfRangeWithTwoUnEvenBuckets") {
|
test("WorksWithOutOfRangeWithTwoUnEvenBuckets") {
|
||||||
// Verify that out of range works with two un even buckets
|
// Verify that out of range works with two un even buckets
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01))
|
val rdd = sc.parallelize(Seq(10.01, -0.01))
|
||||||
val buckets: Array[Double] = Array(0.0, 4.0, 10.0)
|
val buckets = Array(0.0, 4.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(0, 0)
|
val expectedHistogramResults = Array(0, 0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksInRangeWithTwoBuckets") {
|
test("WorksInRangeWithTwoBuckets") {
|
||||||
// Make sure that it works with two equally spaced buckets and elements in each
|
// Make sure that it works with two equally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 10.0)
|
val buckets = Array(0.0, 5.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(3, 2)
|
val expectedHistogramResults = Array(3, 2)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksInRangeWithTwoBucketsAndNaN") {
|
test("WorksInRangeWithTwoBucketsAndNaN") {
|
||||||
// Make sure that it works with two equally spaced buckets and elements in each
|
// Make sure that it works with two equally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6, Double.NaN))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6, Double.NaN))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 10.0)
|
val buckets = Array(0.0, 5.0, 10.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val histogramResults2: Array[Long] = rdd.histogram(buckets, true)
|
val histogramResults2 = rdd.histogram(buckets, true)
|
||||||
val expectedHistogramResults: Array[Long] = Array(3, 2)
|
val expectedHistogramResults = Array(3, 2)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramResults2 === expectedHistogramResults)
|
assert(histogramResults2 === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksInRangeWithTwoUnevenBuckets") {
|
test("WorksInRangeWithTwoUnevenBuckets") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 5, 6))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 5, 6))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0)
|
val buckets = Array(0.0, 5.0, 11.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(3, 2)
|
val expectedHistogramResults = Array(3, 2)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksMixedRangeWithTwoUnevenBuckets") {
|
test("WorksMixedRangeWithTwoUnevenBuckets") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01))
|
val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0)
|
val buckets = Array(0.0, 5.0, 11.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4, 3)
|
val expectedHistogramResults = Array(4, 3)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksMixedRangeWithFourUnevenBuckets") {
|
test("WorksMixedRangeWithFourUnevenBuckets") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
||||||
200.0, 200.1))
|
200.0, 200.1))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0)
|
val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 3)
|
val expectedHistogramResults = Array(4, 2, 1, 3)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksMixedRangeWithUnevenBucketsAndNaN") {
|
test("WorksMixedRangeWithUnevenBucketsAndNaN") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
||||||
200.0, 200.1, Double.NaN))
|
200.0, 200.1, Double.NaN))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0)
|
val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 3)
|
val expectedHistogramResults = Array(4, 2, 1, 3)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
// Make sure this works with a NaN end bucket
|
// Make sure this works with a NaN end bucket
|
||||||
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRange") {
|
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRange") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
||||||
200.0, 200.1, Double.NaN))
|
200.0, 200.1, Double.NaN))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
|
val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 2, 3)
|
val expectedHistogramResults = Array(4, 2, 1, 2, 3)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
// Make sure this works with a NaN end bucket and an inifity
|
// Make sure this works with a NaN end bucket and an inifity
|
||||||
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRangeAndInfity") {
|
test("WorksMixedRangeWithUnevenBucketsAndNaNAndNaNRangeAndInfity") {
|
||||||
// Make sure that it works with two unequally spaced buckets and elements in each
|
// Make sure that it works with two unequally spaced buckets and elements in each
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
val rdd = sc.parallelize(Seq(-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0,
|
||||||
200.0, 200.1, 1.0/0.0, -1.0/0.0, Double.NaN))
|
200.0, 200.1, 1.0/0.0, -1.0/0.0, Double.NaN))
|
||||||
val buckets: Array[Double] = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
|
val buckets = Array(0.0, 5.0, 11.0, 12.0, 200.0, Double.NaN)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4, 2, 1, 2, 4)
|
val expectedHistogramResults = Array(4, 2, 1, 2, 4)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
test("WorksWithOutOfRangeWithInfiniteBuckets") {
|
test("WorksWithOutOfRangeWithInfiniteBuckets") {
|
||||||
// Verify that out of range works with two buckets
|
// Verify that out of range works with two buckets
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(10.01, -0.01, Double.NaN))
|
val rdd = sc.parallelize(Seq(10.01, -0.01, Double.NaN))
|
||||||
val buckets: Array[Double] = Array(-1.0/0.0 , 0.0, 1.0/0.0)
|
val buckets = Array(-1.0/0.0 , 0.0, 1.0/0.0)
|
||||||
val histogramResults: Array[Long] = rdd.histogram(buckets)
|
val histogramResults = rdd.histogram(buckets)
|
||||||
val expectedHistogramResults: Array[Long] = Array(1, 1)
|
val expectedHistogramResults = Array(1, 1)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
}
|
}
|
||||||
// Test the failure mode with an invalid bucket array
|
// Test the failure mode with an invalid bucket array
|
||||||
test("ThrowsExceptionOnInvalidBucketArray") {
|
test("ThrowsExceptionOnInvalidBucketArray") {
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1.0))
|
val rdd = sc.parallelize(Seq(1.0))
|
||||||
// Empty array
|
// Empty array
|
||||||
intercept[IllegalArgumentException] {
|
intercept[IllegalArgumentException] {
|
||||||
val buckets: Array[Double] = Array.empty[Double]
|
val buckets = Array.empty[Double]
|
||||||
val result = rdd.histogram(buckets)
|
val result = rdd.histogram(buckets)
|
||||||
}
|
}
|
||||||
// Single element array
|
// Single element array
|
||||||
intercept[IllegalArgumentException] {
|
intercept[IllegalArgumentException] {
|
||||||
val buckets: Array[Double] = Array(1.0)
|
val buckets = Array(1.0)
|
||||||
val result = rdd.histogram(buckets)
|
val result = rdd.histogram(buckets)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -187,49 +187,49 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
|
||||||
// Test automatic histogram function
|
// Test automatic histogram function
|
||||||
test("WorksWithoutBucketsBasic") {
|
test("WorksWithoutBucketsBasic") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 4))
|
||||||
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4)
|
val expectedHistogramResults = Array(4)
|
||||||
val expectedHistogramBuckets: Array[Double] = Array(1.0, 4.0)
|
val expectedHistogramBuckets = Array(1.0, 4.0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramBuckets === expectedHistogramBuckets)
|
assert(histogramBuckets === expectedHistogramBuckets)
|
||||||
}
|
}
|
||||||
// Test automatic histogram function with a single element
|
// Test automatic histogram function with a single element
|
||||||
test("WorksWithoutBucketsBasicSingleElement") {
|
test("WorksWithoutBucketsBasicSingleElement") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1))
|
val rdd = sc.parallelize(Seq(1))
|
||||||
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
||||||
val expectedHistogramResults: Array[Long] = Array(1)
|
val expectedHistogramResults = Array(1)
|
||||||
val expectedHistogramBuckets: Array[Double] = Array(1.0, 1.0)
|
val expectedHistogramBuckets = Array(1.0, 1.0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramBuckets === expectedHistogramBuckets)
|
assert(histogramBuckets === expectedHistogramBuckets)
|
||||||
}
|
}
|
||||||
// Test automatic histogram function with a single element
|
// Test automatic histogram function with a single element
|
||||||
test("WorksWithoutBucketsBasicNoRange") {
|
test("WorksWithoutBucketsBasicNoRange") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 1, 1, 1))
|
val rdd = sc.parallelize(Seq(1, 1, 1, 1))
|
||||||
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
val (histogramBuckets, histogramResults) = rdd.histogram(1)
|
||||||
val expectedHistogramResults: Array[Long] = Array(4)
|
val expectedHistogramResults = Array(4)
|
||||||
val expectedHistogramBuckets: Array[Double] = Array(1.0, 1.0)
|
val expectedHistogramBuckets = Array(1.0, 1.0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramBuckets === expectedHistogramBuckets)
|
assert(histogramBuckets === expectedHistogramBuckets)
|
||||||
}
|
}
|
||||||
test("WorksWithoutBucketsBasicTwo") {
|
test("WorksWithoutBucketsBasicTwo") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2, 3, 4))
|
val rdd = sc.parallelize(Seq(1, 2, 3, 4))
|
||||||
val (histogramBuckets, histogramResults) = rdd.histogram(2)
|
val (histogramBuckets, histogramResults) = rdd.histogram(2)
|
||||||
val expectedHistogramResults: Array[Long] = Array(2, 2)
|
val expectedHistogramResults = Array(2, 2)
|
||||||
val expectedHistogramBuckets: Array[Double] = Array(1.0, 2.5, 4.0)
|
val expectedHistogramBuckets = Array(1.0, 2.5, 4.0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramBuckets === expectedHistogramBuckets)
|
assert(histogramBuckets === expectedHistogramBuckets)
|
||||||
}
|
}
|
||||||
test("WorksWithoutBucketsWithMoreRequestedThanElements") {
|
test("WorksWithoutBucketsWithMoreRequestedThanElements") {
|
||||||
// Verify the basic case of one bucket and all elements in that bucket works
|
// Verify the basic case of one bucket and all elements in that bucket works
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 2))
|
val rdd = sc.parallelize(Seq(1, 2))
|
||||||
val (histogramBuckets, histogramResults) = rdd.histogram(10)
|
val (histogramBuckets, histogramResults) = rdd.histogram(10)
|
||||||
val expectedHistogramResults: Array[Long] =
|
val expectedHistogramResults =
|
||||||
Array(1, 0, 0, 0, 0, 0, 0, 0, 0, 1)
|
Array(1, 0, 0, 0, 0, 0, 0, 0, 0, 1)
|
||||||
val expectedHistogramBuckets: Array[Double] =
|
val expectedHistogramBuckets =
|
||||||
Array(1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0)
|
Array(1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0)
|
||||||
assert(histogramResults === expectedHistogramResults)
|
assert(histogramResults === expectedHistogramResults)
|
||||||
assert(histogramBuckets === expectedHistogramBuckets)
|
assert(histogramBuckets === expectedHistogramBuckets)
|
||||||
|
@ -239,12 +239,12 @@ class DoubleRDDSuite extends FunSuite with SharedSparkContext {
|
||||||
test("ThrowsExceptionOnInvalidRDDs") {
|
test("ThrowsExceptionOnInvalidRDDs") {
|
||||||
// infinity
|
// infinity
|
||||||
intercept[UnsupportedOperationException] {
|
intercept[UnsupportedOperationException] {
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, 1.0/0.0))
|
val rdd = sc.parallelize(Seq(1, 1.0/0.0))
|
||||||
val result = rdd.histogram(1)
|
val result = rdd.histogram(1)
|
||||||
}
|
}
|
||||||
// NaN
|
// NaN
|
||||||
intercept[UnsupportedOperationException] {
|
intercept[UnsupportedOperationException] {
|
||||||
val rdd: RDD[Double] = sc.parallelize(Seq(1, Double.NaN))
|
val rdd = sc.parallelize(Seq(1, Double.NaN))
|
||||||
val result = rdd.histogram(1)
|
val result = rdd.histogram(1)
|
||||||
}
|
}
|
||||||
// Empty
|
// Empty
|
||||||
|
|
Loading…
Reference in a new issue