[SPARK-10875] [MLLIB] Computed covariance matrix should be symmetric
Compute upper triangular values of the covariance matrix, then copy to lower triangular values. Author: Nick Pritchard <nicholas.pritchard@falkonry.com> Closes #8940 from pnpritchard/SPARK-10875.
This commit is contained in:
parent
5410747a84
commit
5994cfe812
|
@ -357,9 +357,11 @@ class RowMatrix @Since("1.0.0") (
|
|||
var alpha = 0.0
|
||||
while (i < n) {
|
||||
alpha = m / m1 * mean(i)
|
||||
j = 0
|
||||
j = i
|
||||
while (j < n) {
|
||||
G(i, j) = G(i, j) / m1 - alpha * mean(j)
|
||||
val Gij = G(i, j) / m1 - alpha * mean(j)
|
||||
G(i, j) = Gij
|
||||
G(j, i) = Gij
|
||||
j += 1
|
||||
}
|
||||
i += 1
|
||||
|
|
|
@ -24,6 +24,7 @@ import breeze.linalg.{DenseVector => BDV, DenseMatrix => BDM, norm => brzNorm, s
|
|||
|
||||
import org.apache.spark.SparkFunSuite
|
||||
import org.apache.spark.mllib.linalg.{Matrices, Vectors, Vector}
|
||||
import org.apache.spark.mllib.random.RandomRDDs
|
||||
import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext}
|
||||
|
||||
class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext {
|
||||
|
@ -255,6 +256,23 @@ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext {
|
|||
assert(closeToZero(abs(expected.r) - abs(rOnly.R.toBreeze.asInstanceOf[BDM[Double]])))
|
||||
}
|
||||
}
|
||||
|
||||
test("compute covariance") {
|
||||
for (mat <- Seq(denseMat, sparseMat)) {
|
||||
val result = mat.computeCovariance()
|
||||
val expected = breeze.linalg.cov(mat.toBreeze())
|
||||
assert(closeToZero(abs(expected) - abs(result.toBreeze.asInstanceOf[BDM[Double]])))
|
||||
}
|
||||
}
|
||||
|
||||
test("covariance matrix is symmetric (SPARK-10875)") {
|
||||
val rdd = RandomRDDs.normalVectorRDD(sc, 100, 10, 0, 0)
|
||||
val matrix = new RowMatrix(rdd)
|
||||
val cov = matrix.computeCovariance()
|
||||
for (i <- 0 until cov.numRows; j <- 0 until i) {
|
||||
assert(cov(i, j) === cov(j, i))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class RowMatrixClusterSuite extends SparkFunSuite with LocalClusterSparkContext {
|
||||
|
|
Loading…
Reference in a new issue