From 7e2ed40d58a1b4a75b6d596383026bc952215492 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 20 May 2020 19:18:05 +0900
Subject: [PATCH] [SPARK-31759][DEPLOY] Support configurable max number of
 rotate logs for spark daemons
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

in `spark-daemon.sh`, `spark_rotate_log()` accepts `$2` as a custom setting for the number of maximum rotate log files, but this part of code is actually never used.

This PR adds `SPARK_LOG_MAX_FILES` environment variable to represent the maximum log files of Spark daemons can rotate to.

### Why are the changes needed?

the logs files that all spark daemons are hardcoded as 5, but it supposed to be configurable

### Does this PR introduce _any_ user-facing change?

yes, SPARK_LOG_MAX_FILES is added to represent the maximum log files of Spark daemons can rotate to.

### How was this patch tested?

verify locally for the added shell script:

```shell
 kentyaohulk  ~  SPARK_LOG_MAX_FILES=1 sh test.sh
1
 kentyaohulk  ~  SPARK_LOG_MAX_FILES=a sh test.sh
Error: SPARK_LOG_MAX_FILES must be a postive number
 ✘ kentyaohulk  ~  SPARK_LOG_MAX_FILES=b sh test.sh
Error: SPARK_LOG_MAX_FILES must be a postive number
 ✘ kentyaohulk  ~  SPARK_LOG_MAX_FILES=-1 sh test.sh
Error: SPARK_LOG_MAX_FILES must be a postive number
 ✘ kentyaohulk  ~  sh test.sh
5
 ✘ kentyaohulk  ~  cat test.sh
#!/bin/bash

if [[ -z ${SPARK_LOG_MAX_FILES} ]] ; then
      num=5
elif [[ ${SPARK_LOG_MAX_FILES} -gt 0 ]]; then
      num=${SPARK_LOG_MAX_FILES}
else
    echo "Error: SPARK_LOG_MAX_FILES must be a postive number"
    exit -1
fi
```

Closes #28580 from yaooqinn/SPARK-31759.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 conf/spark-env.sh.template |  1 +
 sbin/spark-daemon.sh       | 13 ++++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template
index df39ad8b0d..3c003f45ed 100755
--- a/conf/spark-env.sh.template
+++ b/conf/spark-env.sh.template
@@ -62,6 +62,7 @@
 # Generic options for the daemons used in the standalone deploy mode
 # - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
 # - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
+# - SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5.
 # - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
 # - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
 # - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh
index 81f2fd40a7..e563f7bff1 100755
--- a/sbin/spark-daemon.sh
+++ b/sbin/spark-daemon.sh
@@ -23,6 +23,7 @@
 #
 #   SPARK_CONF_DIR  Alternate conf dir. Default is ${SPARK_HOME}/conf.
 #   SPARK_LOG_DIR   Where log files are stored. ${SPARK_HOME}/logs by default.
+#   SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5.
 #   SPARK_MASTER    host:path where spark code should be rsync'd from
 #   SPARK_PID_DIR   The pid files are stored. /tmp by default.
 #   SPARK_IDENT_STRING   A string representing this instance of spark. $USER by default
@@ -74,10 +75,16 @@ shift
 spark_rotate_log ()
 {
     log=$1;
-    num=5;
-    if [ -n "$2" ]; then
-	num=$2
+
+    if [[ -z ${SPARK_LOG_MAX_FILES} ]]; then
+      num=5
+    elif [[ ${SPARK_LOG_MAX_FILES} -gt 0 ]]; then
+      num=${SPARK_LOG_MAX_FILES}
+    else
+      echo "Error: SPARK_LOG_MAX_FILES must be a positive number, but got ${SPARK_LOG_MAX_FILES}"
+      exit -1
     fi
+
     if [ -f "$log" ]; then # rotate logs
 	while [ $num -gt 1 ]; do
 	    prev=`expr $num - 1`