From 11fcbc73cbcbb1bdf5ba5d90eba0aba1edebb15d Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 5 Jul 2021 09:17:06 +0900 Subject: [PATCH] [SPARK-36007][INFRA] Failed to run benchmark in GA ### What changes were proposed in this pull request? When I'm running the benchmark in GA, I met the below error. https://github.com/pingsutw/spark/runs/2867617238?check_suite_focus=true ``` java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.j ava:1692)java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) 21/06/20 07:40:02 ERROR SparkContext: Error initializing SparkContext.java.lang.AssertionError: assertion failed: spark.test.home is not set! at scala.Predef$.assert(Predef.scala:223) at org.apache.spark.deploy.worker.Worker. (Worker.scala:148) at org.apache.spark.deploy.worker.Worker$.startRpcEnvAndEndpoint(Worker.scala:954) at org.apache.spark.deploy.LocalSparkCluster.$anonfun$start$2(LocalSparkCluster.scala:68) at org.apache.spark.deploy.LocalSparkCluster.$anonfun$start$2$adapted(LocalSparkCluster.scala:65) at scala.collection.immutable.Range.foreach(Range.scala:158) at org.apache.spark.deploy.LocalSparkCluster.start(LocalSparkCluster.scala:65) at org.apache.spark.SparkContext$.org$apache$spark$SparkContext$$createTaskScheduler(SparkContext.scala:2954) at org.apache.spark.SparkContext.(SparkContext.scala:559) at org.apache.spark.SparkContext. (SparkContext.scala:137) at org.apache.spark.serializer.KryoSerializerBenchmark$.createSparkContext(KryoSerializerBenchmark.scala:86) at org.apache.spark.serializer.KryoSerializerBenchmark$.sc$lzycompute$1(KryoSerializerBenchmark.scala:58) at org.apache.spark.serializer.KryoSerializerBenchmark$.sc$1(KryoSerializerBenchmark.scala:58) at org.apache.spark.serializer.KryoSerializerBenchmark$.$anonfun$run$3(KryoSerializerBenchmark.scala:63) ``` ### Why are the changes needed? Set `spark.test.home` in the benchmark workflow. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Rerun the benchmark in my fork. https://github.com/pingsutw/spark/actions/runs/996067851 Closes #33203 from pingsutw/SPARK-36007. Lead-authored-by: Kevin Su Co-authored-by: Kevin Su Signed-off-by: Hyukjin Kwon --- .github/workflows/benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 76ae152f23..9a599b9ca3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -48,6 +48,8 @@ jobs: SPARK_BENCHMARK_CUR_SPLIT: ${{ matrix.split }} SPARK_GENERATE_BENCHMARK_FILES: 1 SPARK_LOCAL_IP: localhost + # To prevent spark.test.home not being set. See more detail in SPARK-36007. + SPARK_HOME: ${{ github.workspace }} steps: - name: Checkout Spark repository uses: actions/checkout@v2