diff --git a/dev/lint-scala b/dev/lint-scala index c676dfdf4f..9c701ab463 100755 --- a/dev/lint-scala +++ b/dev/lint-scala @@ -20,4 +20,4 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)" -"$SCRIPT_DIR/scalastyle" +"$SCRIPT_DIR/scalastyle" "$1" diff --git a/dev/mima b/dev/mima index dc7b085698..f324c5c00a 100755 --- a/dev/mima +++ b/dev/mima @@ -24,7 +24,7 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive" +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)" OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)" diff --git a/dev/run-tests.py b/dev/run-tests.py index aa106aff9e..535c8775fd 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -175,9 +175,11 @@ def run_apache_rat_checks(): run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")]) -def run_scala_style_checks(): +def run_scala_style_checks(build_profiles): set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE") - run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala")]) + profiles = " ".join(build_profiles) + print("[info] Checking Scala style using SBT with these profiles: ", profiles) + run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles]) def run_java_style_checks(build_profiles): @@ -359,7 +361,10 @@ def build_apache_spark(build_tool, hadoop_version): def detect_binary_inop_with_mima(hadoop_version): build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags set_title_and_block("Detecting binary incompatibilities with MiMa", "BLOCK_MIMA") - run_cmd([os.path.join(SPARK_HOME, "dev", "mima")] + build_profiles) + profiles = " ".join(build_profiles) + print("[info] Detecting binary incompatibilities with MiMa using SBT with these profiles: ", + profiles) + run_cmd([os.path.join(SPARK_HOME, "dev", "mima"), profiles]) def run_scala_tests_maven(test_profiles): @@ -582,7 +587,8 @@ def main(): if not changed_files or any(f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files): - run_scala_style_checks() + build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags + run_scala_style_checks(build_profiles) should_run_java_style_checks = False if not changed_files or any(f.endswith(".java") or f.endswith("checkstyle.xml") diff --git a/dev/sbt-checkstyle b/dev/sbt-checkstyle index 1ecad59125..415155fd1b 100755 --- a/dev/sbt-checkstyle +++ b/dev/sbt-checkstyle @@ -17,12 +17,12 @@ # limitations under the License. # -profiles=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver"} +SPARK_PROFILES=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver"} # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. ERRORS=$(echo -e "q\n" \ - | build/sbt ${profiles} checkstyle test:checkstyle \ + | build/sbt ${SPARK_PROFILES} checkstyle test:checkstyle \ | awk '{if($1~/error/)print}' \ ) diff --git a/dev/scalastyle b/dev/scalastyle index d5cebb3943..212ef900eb 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -17,19 +17,15 @@ # limitations under the License. # +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} + # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. ERRORS=$(echo -e "q\n" \ | build/sbt \ - -Pkinesis-asl \ - -Pmesos \ - -Pkubernetes \ - -Pyarn \ - -Phive \ - -Phive-thriftserver \ - -Pspark-ganglia-lgpl \ + ${SPARK_PROFILES} \ -Pdocker-integration-tests \ - -Pkubernetes-integration-tests \ + -Pkubernetes-integration-tests \ scalastyle test:scalastyle \ | awk '{if($1~/error/)print}' \ )