From 7d13ac177ba745e42c406ee8aa1594cc3448dbc6 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 5 Aug 2021 01:31:35 -0700
Subject: [PATCH] [SPARK-36393][BUILD] Try to raise memory for GHA

### What changes were proposed in this pull request?

According to the feedback from GitHub, the change causing memory issue has been rolled back. We can try to raise memory again for GA.

### Why are the changes needed?

Trying higher memory settings for GA. It could speed up the testing time.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

GA

Closes #33623 from viirya/increasing-mem-ga.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 .github/workflows/build_and_test.yml | 2 +-
 build/sbt-launch-lib.bash            | 6 ++++--
 dev/run-tests.py                     | 7 +------
 pom.xml                              | 8 ++++----
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 665586cd3e..62134479d0 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -220,7 +220,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
       SKIP_UNIDOC: true
       SKIP_MIMA: true
-      METASPACE_SIZE: 128m
+      METASPACE_SIZE: 1g
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash
index 7261e019dd..8fb6672bdd 100755
--- a/build/sbt-launch-lib.bash
+++ b/build/sbt-launch-lib.bash
@@ -117,9 +117,11 @@ addDebugger () {
 # so they need not be dicked around with individually.
 get_mem_opts () {
   local mem=${1:-$sbt_default_mem}
-  local codecache=128
+  local codecache=$(( $mem / 8 ))
+  (( $codecache > 128 )) || codecache=128
+  (( $codecache < 2048 )) || codecache=2048
 
-  echo "-Xms256m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
+  echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
 }
 
 require_arg () {
diff --git a/dev/run-tests.py b/dev/run-tests.py
index d34fa1bd35..0f0759bd32 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -268,12 +268,7 @@ def exec_sbt(sbt_args=()):
     """Will call SBT in the current directory with the list of mvn_args passed
     in and returns the subprocess for any further processing"""
 
-    sbt_cmd = [os.path.join(SPARK_HOME, "build", "sbt")]
-
-    if "GITHUB_ACTIONS" in os.environ:
-        sbt_cmd = sbt_cmd + ['-mem', '2300']
-
-    sbt_cmd = sbt_cmd + sbt_args
+    sbt_cmd = [os.path.join(SPARK_HOME, "build", "sbt")] + sbt_args
 
     sbt_output_filter = re.compile(b"^.*[info].*Resolving" + b"|" +
                                    b"^.*[warn].*Merging" + b"|" +
diff --git a/pom.xml b/pom.xml
index 8f82d1e668..4413351fe1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2617,8 +2617,8 @@
             </args>
             <jvmArgs>
               <jvmArg>-Xss128m</jvmArg>
-              <jvmArg>-Xms1024m</jvmArg>
-              <jvmArg>-Xmx3200m</jvmArg>
+              <jvmArg>-Xms4g</jvmArg>
+              <jvmArg>-Xmx4g</jvmArg>
               <jvmArg>-XX:MaxMetaspaceSize=2g</jvmArg>
               <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
             </jvmArgs>
@@ -2668,7 +2668,7 @@
               <include>**/*Suite.java</include>
             </includes>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-            <argLine>-ea -Xmx3200m -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
             <environmentVariables>
               <!--
                 Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
@@ -2719,7 +2719,7 @@
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
             <junitxml>.</junitxml>
             <filereports>SparkTestSuite.txt</filereports>
-            <argLine>-ea -Xmx3200m -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
             <stderr/>
             <environmentVariables>
               <!--