SPARK-6085 Increase default value for memory overhead
Author: tedyu <yuzhihong@gmail.com> Closes #4836 from tedyu/master and squashes the following commits: d65b495 [tedyu] SPARK-6085 Increase default value for memory overhead 1fdd4df [tedyu] SPARK-6085 Increase default value for memory overhead
This commit is contained in:
parent
76e20a0a03
commit
8d3e2414d4
|
@ -21,7 +21,7 @@ import org.apache.spark.SparkContext
|
|||
|
||||
private[spark] object MemoryUtils {
|
||||
// These defaults copied from YARN
|
||||
val OVERHEAD_FRACTION = 1.07
|
||||
val OVERHEAD_FRACTION = 1.10
|
||||
val OVERHEAD_MINIMUM = 384
|
||||
|
||||
def calculateTotalMemory(sc: SparkContext) = {
|
||||
|
|
|
@ -225,7 +225,7 @@ See the [configuration page](configuration.html) for information on Spark config
|
|||
</tr>
|
||||
<tr>
|
||||
<td><code>spark.mesos.executor.memoryOverhead</code></td>
|
||||
<td>executor memory * 0.07, with minimum of 384</td>
|
||||
<td>executor memory * 0.10, with minimum of 384</td>
|
||||
<td>
|
||||
This value is an additive for <code>spark.executor.memory</code>, specified in MiB,
|
||||
which is used to calculate the total Mesos task memory. A value of <code>384</code>
|
||||
|
|
|
@ -113,7 +113,7 @@ Most of the configs are the same for Spark on YARN as for other deployment modes
|
|||
</tr>
|
||||
<tr>
|
||||
<td><code>spark.yarn.executor.memoryOverhead</code></td>
|
||||
<td>executorMemory * 0.07, with minimum of 384 </td>
|
||||
<td>executorMemory * 0.10, with minimum of 384 </td>
|
||||
<td>
|
||||
The amount of off heap memory (in megabytes) to be allocated per executor. This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to grow with the executor size (typically 6-10%).
|
||||
</td>
|
||||
|
|
|
@ -86,10 +86,10 @@ class YarnSparkHadoopUtil extends SparkHadoopUtil {
|
|||
|
||||
object YarnSparkHadoopUtil {
|
||||
// Additional memory overhead
|
||||
// 7% was arrived at experimentally. In the interest of minimizing memory waste while covering
|
||||
// 10% was arrived at experimentally. In the interest of minimizing memory waste while covering
|
||||
// the common cases. Memory overhead tends to grow with container size.
|
||||
|
||||
val MEMORY_OVERHEAD_FACTOR = 0.07
|
||||
val MEMORY_OVERHEAD_FACTOR = 0.10
|
||||
val MEMORY_OVERHEAD_MIN = 384
|
||||
|
||||
val ANY_HOST = "*"
|
||||
|
|
Loading…
Reference in a new issue