ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-9277] [MLLIB] SparseVector constructor must throw an error when declared number of elements less than array length

Browse source 
Check that SparseVector size is at least as big as the number of indices/values provided. And add tests for constructor checks.

CC MechCoder jkbradley -- I am not sure if a change needs to also happen in the Python API? I didn't see it had any similar checks to begin with, but I don't know it well.

Author: Sean Owen <sowen@cloudera.com>

Closes #7794 from srowen/SPARK-9277 and squashes the following commits:

e8dc31e [Sean Owen] Fix scalastyle
6ffe34a [Sean Owen] Check that SparseVector size is at least as big as the number of indices/values provided. And add tests for constructor checks.
This commit is contained in:
Sean Owen 2015-07-30 09:19:55 -07:00 committed by Xiangrui Meng
parent a6e53a9c8b
commit ed3cb1d21c
2 changed files with 17 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
View file

@ -637,6 +637,8 @@ class SparseVector(
require(indices.length == values.length, "Sparse vectors require that the dimension of the" +
s" indices match the dimension of the values. You provided ${indices.length} indices and " +
s" ${values.length} values.")
require(indices.length <= size, s"You provided ${indices.length} indices and values, " +
s"which exceeds the specified vector size ${size}.")
override def toString: String =
s"($size,${indices.mkString("[", ",", "]")},${values.mkString("[", ",", "]")})"

15
mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
View file

@ -57,6 +57,21 @@ class VectorsSuite extends SparkFunSuite with Logging {
assert(vec.values === values)
}
test("sparse vector construction with mismatched indices/values array") {
intercept[IllegalArgumentException] {
Vectors.sparse(4, Array(1, 2, 3), Array(3.0, 5.0, 7.0, 9.0))
}
intercept[IllegalArgumentException] {
Vectors.sparse(4, Array(1, 2, 3), Array(3.0, 5.0))
}
}
test("sparse vector construction with too many indices vs size") {
intercept[IllegalArgumentException] {
Vectors.sparse(3, Array(1, 2, 3, 4), Array(3.0, 5.0, 7.0, 9.0))
}
}
test("dense to array") {
val vec = Vectors.dense(arr).asInstanceOf[DenseVector]
assert(vec.toArray.eq(arr))