[SPARK-14679][UI] Fix UI DAG visualization OOM.
## What changes were proposed in this pull request? The DAG visualization can cause an OOM when generating the DOT file. This happens because clusters are not correctly deduped by a contains check because they use the default equals implementation. This adds a working equals implementation. ## How was this patch tested? This adds a test suite that checks the new equals implementation. Author: Ryan Blue <blue@apache.org> Closes #12437 from rdblue/SPARK-14679-fix-ui-oom.
This commit is contained in:
parent
7abe9a6578
commit
a3451119d9
|
@ -17,6 +17,8 @@
|
|||
|
||||
package org.apache.spark.ui.scope
|
||||
|
||||
import java.util.Objects
|
||||
|
||||
import scala.collection.mutable
|
||||
import scala.collection.mutable.{ListBuffer, StringBuilder}
|
||||
|
||||
|
@ -72,6 +74,22 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String)
|
|||
def getCachedNodes: Seq[RDDOperationNode] = {
|
||||
_childNodes.filter(_.cached) ++ _childClusters.flatMap(_.getCachedNodes)
|
||||
}
|
||||
|
||||
def canEqual(other: Any): Boolean = other.isInstanceOf[RDDOperationCluster]
|
||||
|
||||
override def equals(other: Any): Boolean = other match {
|
||||
case that: RDDOperationCluster =>
|
||||
(that canEqual this) &&
|
||||
_childClusters == that._childClusters &&
|
||||
id == that.id &&
|
||||
_name == that._name
|
||||
case _ => false
|
||||
}
|
||||
|
||||
override def hashCode(): Int = {
|
||||
val state = Seq(_childClusters, id, _name)
|
||||
state.map(Objects.hashCode).foldLeft(0)((a, b) => 31 * a + b)
|
||||
}
|
||||
}
|
||||
|
||||
private[ui] object RDDOperationGraph extends Logging {
|
||||
|
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.spark.ui.scope
|
||||
|
||||
import org.apache.spark.SparkFunSuite
|
||||
|
||||
class RDDOperationGraphSuite extends SparkFunSuite {
|
||||
test("Test simple cluster equals") {
|
||||
// create a 2-cluster chain with a child
|
||||
val c1 = new RDDOperationCluster("1", "Bender")
|
||||
val c2 = new RDDOperationCluster("2", "Hal")
|
||||
c1.attachChildCluster(c2)
|
||||
c1.attachChildNode(new RDDOperationNode(3, "Marvin", false, "collect!"))
|
||||
|
||||
// create an equal cluster, but without the child node
|
||||
val c1copy = new RDDOperationCluster("1", "Bender")
|
||||
val c2copy = new RDDOperationCluster("2", "Hal")
|
||||
c1copy.attachChildCluster(c2copy)
|
||||
|
||||
assert(c1 == c1copy)
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue