From ff3a737c759b21d166053a0fbb6ad539accb8e49 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 20 Sep 2019 19:41:25 -0700 Subject: [PATCH] [SPARK-29192][TESTS] Extend BenchmarkBase to write JDK9+ results separately ### What changes were proposed in this pull request? This PR aims to extend the existing benchmarks to save JDK9+ result separately. All `core` module benchmark test results are added. I'll run the other test suites in another PR. After regenerating all results, we will check JDK11 performance regressions. ### Why are the changes needed? From Apache Spark 3.0, we support both JDK8 and JDK11. We need to have a way to find the performance regression. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Manually run the benchmark. Closes #25873 from dongjoon-hyun/SPARK-JDK11-PERF. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../CoalescedRDDBenchmark-jdk11-results.txt | 40 +++++++++++++++++ .../KryoBenchmark-jdk11-results.txt | 28 ++++++++++++ .../KryoSerializerBenchmark-jdk11-results.txt | 12 +++++ ...PropertiesCloneBenchmark-jdk11-results.txt | 40 +++++++++++++++++ .../XORShiftRandomBenchmark-jdk11-results.txt | 44 +++++++++++++++++++ .../spark/benchmark/BenchmarkBase.scala | 5 ++- 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt create mode 100644 core/benchmarks/KryoBenchmark-jdk11-results.txt create mode 100644 core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt create mode 100644 core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt create mode 100644 core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt diff --git a/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt b/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt new file mode 100644 index 0000000000..f43e083060 --- /dev/null +++ b/core/benchmarks/CoalescedRDDBenchmark-jdk11-results.txt @@ -0,0 +1,40 @@ +================================================================================================ +Coalesced RDD , large scale +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Coalesced RDD: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +Coalesce Num Partitions: 100 Num Hosts: 1 455 623 151 0.2 4547.4 1.0X +Coalesce Num Partitions: 100 Num Hosts: 5 340 409 65 0.3 3397.1 1.3X +Coalesce Num Partitions: 100 Num Hosts: 10 292 363 95 0.3 2923.3 1.6X +Coalesce Num Partitions: 100 Num Hosts: 20 307 320 11 0.3 3069.8 1.5X +Coalesce Num Partitions: 100 Num Hosts: 40 333 368 55 0.3 3329.1 1.4X +Coalesce Num Partitions: 100 Num Hosts: 80 286 338 63 0.3 2862.5 1.6X +Coalesce Num Partitions: 500 Num Hosts: 1 769 837 59 0.1 7693.5 0.6X +Coalesce Num Partitions: 500 Num Hosts: 5 427 461 31 0.2 4268.5 1.1X +Coalesce Num Partitions: 500 Num Hosts: 10 372 389 27 0.3 3722.2 1.2X +Coalesce Num Partitions: 500 Num Hosts: 20 347 365 31 0.3 3468.5 1.3X +Coalesce Num Partitions: 500 Num Hosts: 40 335 336 1 0.3 3347.3 1.4X +Coalesce Num Partitions: 500 Num Hosts: 80 329 360 49 0.3 3294.5 1.4X +Coalesce Num Partitions: 1000 Num Hosts: 1 1254 1292 47 0.1 12538.6 0.4X +Coalesce Num Partitions: 1000 Num Hosts: 5 518 553 47 0.2 5177.0 0.9X +Coalesce Num Partitions: 1000 Num Hosts: 10 394 432 42 0.3 3937.3 1.2X +Coalesce Num Partitions: 1000 Num Hosts: 20 341 381 44 0.3 3414.4 1.3X +Coalesce Num Partitions: 1000 Num Hosts: 40 313 358 48 0.3 3134.9 1.5X +Coalesce Num Partitions: 1000 Num Hosts: 80 335 360 38 0.3 3347.0 1.4X +Coalesce Num Partitions: 5000 Num Hosts: 1 3937 4066 156 0.0 39375.0 0.1X +Coalesce Num Partitions: 5000 Num Hosts: 5 1413 1453 40 0.1 14133.4 0.3X +Coalesce Num Partitions: 5000 Num Hosts: 10 826 861 49 0.1 8255.2 0.6X +Coalesce Num Partitions: 5000 Num Hosts: 20 542 609 58 0.2 5423.3 0.8X +Coalesce Num Partitions: 5000 Num Hosts: 40 410 470 64 0.2 4101.0 1.1X +Coalesce Num Partitions: 5000 Num Hosts: 80 352 427 69 0.3 3515.3 1.3X +Coalesce Num Partitions: 10000 Num Hosts: 1 7101 7151 54 0.0 71007.4 0.1X +Coalesce Num Partitions: 10000 Num Hosts: 5 2540 2582 59 0.0 25396.2 0.2X +Coalesce Num Partitions: 10000 Num Hosts: 10 1378 1432 48 0.1 13781.4 0.3X +Coalesce Num Partitions: 10000 Num Hosts: 20 829 867 66 0.1 8286.8 0.5X +Coalesce Num Partitions: 10000 Num Hosts: 40 573 630 49 0.2 5730.2 0.8X +Coalesce Num Partitions: 10000 Num Hosts: 80 438 449 9 0.2 4382.5 1.0X + + diff --git a/core/benchmarks/KryoBenchmark-jdk11-results.txt b/core/benchmarks/KryoBenchmark-jdk11-results.txt new file mode 100644 index 0000000000..0afa06519d --- /dev/null +++ b/core/benchmarks/KryoBenchmark-jdk11-results.txt @@ -0,0 +1,28 @@ +================================================================================================ +Benchmark Kryo Unsafe vs safe Serialization +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Benchmark Kryo Unsafe vs safe Serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +basicTypes: Int with unsafe:true 324 329 2 3.1 324.0 1.0X +basicTypes: Long with unsafe:true 353 355 1 2.8 353.0 0.9X +basicTypes: Float with unsafe:true 336 338 1 3.0 336.4 1.0X +basicTypes: Double with unsafe:true 347 347 1 2.9 346.5 0.9X +Array: Int with unsafe:true 4 4 0 265.4 3.8 86.0X +Array: Long with unsafe:true 6 7 0 157.3 6.4 51.0X +Array: Float with unsafe:true 4 4 0 268.8 3.7 87.1X +Array: Double with unsafe:true 6 7 0 157.5 6.3 51.0X +Map of string->Double with unsafe:true 52 52 1 19.3 51.8 6.3X +basicTypes: Int with unsafe:false 357 358 1 2.8 357.2 0.9X +basicTypes: Long with unsafe:false 387 388 0 2.6 387.4 0.8X +basicTypes: Float with unsafe:false 356 357 1 2.8 356.0 0.9X +basicTypes: Double with unsafe:false 371 372 1 2.7 371.0 0.9X +Array: Int with unsafe:false 24 24 0 41.3 24.2 13.4X +Array: Long with unsafe:false 37 38 0 26.8 37.4 8.7X +Array: Float with unsafe:false 11 11 0 94.9 10.5 30.8X +Array: Double with unsafe:false 18 18 0 55.2 18.1 17.9X +Map of string->Double with unsafe:false 55 55 0 18.2 55.1 5.9X + + diff --git a/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt b/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt new file mode 100644 index 0000000000..0e6d326498 --- /dev/null +++ b/core/benchmarks/KryoSerializerBenchmark-jdk11-results.txt @@ -0,0 +1,12 @@ +================================================================================================ +Benchmark KryoPool vs old"pool of 1" implementation +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Benchmark KryoPool vs old"pool of 1" implementation: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +KryoPool:true 6524 9149 NaN 0.0 13047076.5 1.0X +KryoPool:false 12855 16469 663 0.0 25709170.6 0.5X + + diff --git a/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt b/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt new file mode 100644 index 0000000000..1cd08a7748 --- /dev/null +++ b/core/benchmarks/PropertiesCloneBenchmark-jdk11-results.txt @@ -0,0 +1,40 @@ +================================================================================================ +Properties Cloning +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Empty Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +SerializationUtils.clone 0 0 0 0.1 13755.0 1.0X +Utils.cloneProperties 0 0 0 3.5 285.0 48.3X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +System Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +SerializationUtils.clone 0 0 0 0.0 191892.0 1.0X +Utils.cloneProperties 0 0 0 0.2 6027.0 31.8X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Small Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +SerializationUtils.clone 1 1 0 0.0 721334.0 1.0X +Utils.cloneProperties 0 0 0 0.2 5237.0 137.7X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Medium Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +SerializationUtils.clone 3 3 0 0.0 3006554.0 1.0X +Utils.cloneProperties 0 0 0 0.0 27042.0 111.2X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Large Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +SerializationUtils.clone 6 6 0 0.0 5864452.0 1.0X +Utils.cloneProperties 0 0 0 0.0 53760.0 109.1X + + diff --git a/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt b/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt new file mode 100644 index 0000000000..3b6091cc74 --- /dev/null +++ b/core/benchmarks/XORShiftRandomBenchmark-jdk11-results.txt @@ -0,0 +1,44 @@ +================================================================================================ +Pseudo random +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +nextInt: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +java.util.Random 1357 1358 0 73.7 13.6 1.0X +XORShiftRandom 228 228 0 438.0 2.3 5.9X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +nextLong: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +java.util.Random 2718 2719 1 36.8 27.2 1.0X +XORShiftRandom 632 633 0 158.1 6.3 4.3X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +nextDouble: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +java.util.Random 2722 2723 0 36.7 27.2 1.0X +XORShiftRandom 632 632 0 158.3 6.3 4.3X + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +nextGaussian: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +java.util.Random 6979 6979 1 14.3 69.8 1.0X +XORShiftRandom 5183 5183 0 19.3 51.8 1.3X + + +================================================================================================ +hash seed +================================================================================================ + +OpenJDK 64-Bit Server VM 11.0.4+11-LTS on Linux 3.10.0-862.3.2.el7.x86_64 +Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz +Hash seed: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +XORShiftRandom.hashSeed 38 39 1 263.9 3.8 1.0X + + diff --git a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala index a6666db4e9..55e34b32fe 100644 --- a/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala +++ b/core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala @@ -21,6 +21,7 @@ import java.io.{File, FileOutputStream, OutputStream} /** * A base class for generate benchmark results to a file. + * For JDK9+, JDK major version number is added to the file names to distingush the results. */ abstract class BenchmarkBase { var output: Option[OutputStream] = None @@ -43,7 +44,9 @@ abstract class BenchmarkBase { def main(args: Array[String]): Unit = { val regenerateBenchmarkFiles: Boolean = System.getenv("SPARK_GENERATE_BENCHMARK_FILES") == "1" if (regenerateBenchmarkFiles) { - val resultFileName = s"${this.getClass.getSimpleName.replace("$", "")}-results.txt" + val version = System.getProperty("java.version").split("\\D+")(0).toInt + val jdkString = if (version > 8) s"-jdk$version" else "" + val resultFileName = s"${this.getClass.getSimpleName.replace("$", "")}$jdkString-results.txt" val file = new File(s"benchmarks/$resultFileName") if (!file.exists()) { file.createNewFile()