SPARK-9545, SPARK-9547: Use Maven in PRB if title contains "[test-maven]"
This is just some small glue code to actually make use of the AMPLAB_JENKINS_BUILD_TOOL switch. As far as I can tell, we actually don't currently use the Maven support in the tool even though it exists. This patch switches to Maven when the PR title contains "test-maven". There are a few small other pieces of cleanup in the patch as well. Author: Patrick Wendell <patrick@databricks.com> Closes #7878 from pwendell/maven-tests.
This commit is contained in:
parent
8d2ab75d3b
commit
35e896a79b
|
@ -164,8 +164,9 @@ pr_message=""
|
|||
current_pr_head="`git rev-parse HEAD`"
|
||||
|
||||
echo "HEAD: `git rev-parse HEAD`"
|
||||
echo "GHPRB: $ghprbActualCommit"
|
||||
echo "SHA1: $sha1"
|
||||
echo "\$ghprbActualCommit: $ghprbActualCommit"
|
||||
echo "\$sha1: $sha1"
|
||||
echo "\$ghprbPullTitle: $ghprbPullTitle"
|
||||
|
||||
# Run pull request tests
|
||||
for t in "${PR_TESTS[@]}"; do
|
||||
|
@ -189,6 +190,19 @@ done
|
|||
{
|
||||
# Marks this build is a pull request build.
|
||||
export AMP_JENKINS_PRB=true
|
||||
if [[ $ghprbPullTitle == *"test-maven"* ]]; then
|
||||
export AMPLAB_JENKINS_BUILD_TOOL="maven"
|
||||
fi
|
||||
if [[ $ghprbPullTitle == *"test-hadoop1.0"* ]]; then
|
||||
export AMPLAB_JENKINS_BUILD_PROFILE="hadoop1.0"
|
||||
elif [[ $ghprbPullTitle == *"test-hadoop2.0"* ]]; then
|
||||
export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.0"
|
||||
elif [[ $ghprbPullTitle == *"test-hadoop2.2"* ]]; then
|
||||
export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.2"
|
||||
elif [[ $ghprbPullTitle == *"test-hadoop2.3"* ]]; then
|
||||
export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.3"
|
||||
fi
|
||||
|
||||
timeout "${TESTS_TIMEOUT}" ./dev/run-tests
|
||||
test_result="$?"
|
||||
|
||||
|
|
|
@ -21,6 +21,7 @@ from __future__ import print_function
|
|||
import itertools
|
||||
from optparse import OptionParser
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
|
@ -239,11 +240,32 @@ def build_spark_documentation():
|
|||
os.chdir(SPARK_HOME)
|
||||
|
||||
|
||||
def get_zinc_port():
|
||||
"""
|
||||
Get a randomized port on which to start Zinc
|
||||
"""
|
||||
return random.randrange(3030, 4030)
|
||||
|
||||
|
||||
def kill_zinc_on_port(zinc_port):
|
||||
"""
|
||||
Kill the Zinc process running on the given port, if one exists.
|
||||
"""
|
||||
cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN "
|
||||
"| awk '{ print $2; }' | xargs kill") % zinc_port
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
def exec_maven(mvn_args=()):
|
||||
"""Will call Maven in the current directory with the list of mvn_args passed
|
||||
in and returns the subprocess for any further processing"""
|
||||
|
||||
run_cmd([os.path.join(SPARK_HOME, "build", "mvn")] + mvn_args)
|
||||
zinc_port = get_zinc_port()
|
||||
os.environ["ZINC_PORT"] = "%s" % zinc_port
|
||||
zinc_flag = "-DzincPort=%s" % zinc_port
|
||||
flags = [os.path.join(SPARK_HOME, "build", "mvn"), "--force", zinc_flag]
|
||||
run_cmd(flags + mvn_args)
|
||||
kill_zinc_on_port(zinc_port)
|
||||
|
||||
|
||||
def exec_sbt(sbt_args=()):
|
||||
|
@ -514,7 +536,9 @@ def main():
|
|||
build_apache_spark(build_tool, hadoop_version)
|
||||
|
||||
# backwards compatibility checks
|
||||
detect_binary_inop_with_mima()
|
||||
if build_tool == "sbt":
|
||||
# Note: compatiblity tests only supported in sbt for now
|
||||
detect_binary_inop_with_mima()
|
||||
|
||||
# run the test suites
|
||||
run_scala_tests(build_tool, hadoop_version, test_modules)
|
||||
|
|
Loading…
Reference in a new issue