[SPARK-4123][Project Infra]: Show new dependencies added in pull requests
Starting work on this, but need to find a way to ensure that, after doing a checkout from `apache/master`, we can successfully return to the current checkout. I believe that `git rev-parse HEAD` will get me what I want, but pushing this PR up to test what the Jenkins boxes are seeing.
Author: Brennon York <brennon.york@capitalone.com>
Closes #5093 from brennonyork/SPARK-4123 and squashes the following commits:
42e243e [Brennon York] moved starting test output to before pr tests, fixed indentation, changed mvn call to build/mvn
dadd941 [Brennon York] reverted assembly pom, put the regular test suite back in play
7aa1dee [Brennon York] set new dendencies into a <code> block, removed the bash debugging flag
0074566 [Brennon York] fixed minor echo issue with quotes
e229802 [Brennon York] updated to print the new dependency found
27bb9b5 [Brennon York] changed the assembly pom to test whether the pr test will pick up new deps
5375ad8 [Brennon York] git output to dev null
9bce980 [Brennon York] ensure both gate files exist
8f3c4b4 [Brennon York] updated to reflect the correct pushed in HEAD variable
2bc7b27 [Brennon York] added a pom gate check
a18db71 [Brennon York] full test of new deps script
ea170de [Brennon York] dont let mvn execute tests
f70d8cd [Brennon York] testing mvn with package
62ffd65 [Brennon York] updated dependency output message and changed compile to package given the jenkins failure output
04747e4 [Brennon York] adding simple mvn statement to see if command executes and prints compile output
87f9bea [Brennon York] added -x flag with bash to get insight into what is executing and what isnt
9e87208 [Brennon York] added set blocks to catch any non-zero exit codes and updated output
6b3042b [Brennon York] removed excess git checkout print statements
4077d46 [Brennon York] Merge remote-tracking branch 'upstream/master' into SPARK-4123
2bb5527 [Brennon York] added echo statement so jenkins logs which pr tests are running
d027f8f [Brennon York] proper piping of unnecessary stderr and stdout
6e2890d [Brennon York] updated test output newlines
d9f6f7f [Brennon York] removed echo
bad9a3a [Brennon York] added back the new deps test
e9e3ad1 [Brennon York] removed escapes for quotes
97e5cfb [Brennon York] commenting out new deps script
17379a5 [Brennon York] Merge remote-tracking branch 'upstream/master' into SPARK-4123
56f74a8 [Brennon York] updated the unop for ensuring a test is available
f2abc8c [Brennon York] removed the git checkout
6912584 [Brennon York] added this_mssg echo output
c610d42 [Brennon York] removed the error to dev/null
b98f78c [Brennon York] added the removed deps and echo output for jenkins testing
291a8fe [Brennon York] updated location of maven binary
126ce61 [Brennon York] removing new deps test to isolate why jenkins isn't posting messages
f8011d8 [Brennon York] minor updates and style changes
63a35c9 [Brennon York] updated new dependencies test
dae7ba8 [Brennon York] Capturing output directly from dependency builds
94d3547 [Brennon York] adding the new dependencies script into the test mix
2bca3c3 [Brennon York] added a git checkout 'git rev-parse HEAD' to the end of each pr test
ae83b90 [Brennon York] removed jenkins tests to grab some values from the jenkins box
4110993 [Brennon York] beginning work on pr test to add new dependencies
This commit is contained in:
Brennon York
Sean Owen
parent
5eef00d0c6
commit
55153f5c14
|
|
@ -55,13 +55,14 @@ TESTS_TIMEOUT="120m" # format: http://linux.die.net/man/1/timeout
|
|||
# To write a PR test:
|
||||
#+ * the file must reside within the dev/tests directory
|
||||
#+ * be an executable bash script
|
||||
#+ * accept two arguments on the command line, the first being the Github PR long commit
|
||||
#+ hash and the second the Github SHA1 hash
|
||||
#+ * accept three arguments on the command line, the first being the Github PR long commit
|
||||
#+ hash, the second the Github SHA1 hash, and the final the current PR hash
|
||||
#+ * and, lastly, return string output to be included in the pr message output that will
|
||||
#+ be posted to Github
|
||||
PR_TESTS=(
|
||||
"pr_merge_ability"
|
||||
"pr_public_classes"
|
||||
"pr_new_dependencies"
|
||||
)
|
||||
|
||||
function post_message () {
|
||||
|
|
@ -146,34 +147,38 @@ function send_archived_logs () {
|
|||
fi
|
||||
}
|
||||
|
||||
# Environment variable to capture PR test output
|
||||
pr_message=""
|
||||
|
||||
# Run pull request tests
|
||||
for t in "${PR_TESTS[@]}"; do
|
||||
this_test="${FWDIR}/dev/tests/${t}.sh"
|
||||
# Ensure the test is a file and is executable
|
||||
if [ -x "$this_test" ]; then
|
||||
echo "ghprb: $ghprbActualCommit sha1: $sha1"
|
||||
this_mssg="`bash \"${this_test}\" \"${ghprbActualCommit}\" \"${sha1}\" 2>/dev/null`"
|
||||
# Check if this is the merge test as we submit that note *before* and *after*
|
||||
# the tests run
|
||||
[ "$t" == "pr_merge_ability" ] && merge_note="${this_mssg}"
|
||||
pr_message="${pr_message}\n${this_mssg}"
|
||||
fi
|
||||
done
|
||||
|
||||
# post start message
|
||||
{
|
||||
start_message="\
|
||||
[Test build ${BUILD_DISPLAY_NAME} has started](${BUILD_URL}consoleFull) for \
|
||||
PR $ghprbPullId at commit [\`${SHORT_COMMIT_HASH}\`](${COMMIT_URL})."
|
||||
|
||||
start_message="${start_message}\n${merge_note}"
|
||||
|
||||
post_message "$start_message"
|
||||
}
|
||||
|
||||
# Environment variable to capture PR test output
|
||||
pr_message=""
|
||||
# Ensure we save off the current HEAD to revert to
|
||||
current_pr_head="`git rev-parse HEAD`"
|
||||
|
||||
# Run pull request tests
|
||||
for t in "${PR_TESTS[@]}"; do
|
||||
this_test="${FWDIR}/dev/tests/${t}.sh"
|
||||
# Ensure the test can be found and is a file
|
||||
if [ -f "${this_test}" ]; then
|
||||
echo "Running test: $t"
|
||||
this_mssg="$(bash "${this_test}" "${ghprbActualCommit}" "${sha1}" "${current_pr_head}")"
|
||||
# Check if this is the merge test as we submit that note *before* and *after*
|
||||
# the tests run
|
||||
[ "$t" == "pr_merge_ability" ] && merge_note="${this_mssg}"
|
||||
pr_message="${pr_message}\n${this_mssg}"
|
||||
# Ensure, after each test, that we're back on the current PR
|
||||
git checkout -f "${current_pr_head}" &>/dev/null
|
||||
else
|
||||
echo "Cannot find test ${this_test}."
|
||||
fi
|
||||
done
|
||||
|
||||
# run tests
|
||||
{
|
||||
timeout "${TESTS_TIMEOUT}" ./dev/run-tests
|
||||
|
|
@ -222,7 +227,7 @@ done
|
|||
PR $ghprbPullId at commit [\`${SHORT_COMMIT_HASH}\`](${COMMIT_URL})."
|
||||
|
||||
result_message="${result_message}\n${test_result_note}"
|
||||
result_message="${result_message}\n${pr_message}"
|
||||
result_message="${result_message}${pr_message}"
|
||||
|
||||
post_message "$result_message"
|
||||
}
|
||||
|
|
|
|||
117
dev/tests/pr_new_dependencies.sh
Executable file
117
dev/tests/pr_new_dependencies.sh
Executable file
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# This script follows the base format for testing pull requests against
|
||||
# another branch and returning results to be published. More details can be
|
||||
# found at dev/run-tests-jenkins.
|
||||
#
|
||||
# Arg1: The Github Pull Request Actual Commit
|
||||
#+ known as `ghprbActualCommit` in `run-tests-jenkins`
|
||||
# Arg2: The SHA1 hash
|
||||
#+ known as `sha1` in `run-tests-jenkins`
|
||||
# Arg3: Current PR Commit Hash
|
||||
#+ the PR hash for the current commit
|
||||
#
|
||||
|
||||
ghprbActualCommit="$1"
|
||||
sha1="$2"
|
||||
current_pr_head="$3"
|
||||
|
||||
MVN_BIN="build/mvn"
|
||||
CURR_CP_FILE="my-classpath.txt"
|
||||
MASTER_CP_FILE="master-classpath.txt"
|
||||
|
||||
# First switch over to the master branch
|
||||
git checkout master &>/dev/null
|
||||
# Find and copy all pom.xml files into a *.gate file that we can check
|
||||
# against through various `git` changes
|
||||
find -name "pom.xml" -exec cp {} {}.gate \;
|
||||
# Switch back to the current PR
|
||||
git checkout "${current_pr_head}" &>/dev/null
|
||||
|
||||
# Check if any *.pom files from the current branch are different from the master
|
||||
difference_q=""
|
||||
for p in $(find -name "pom.xml"); do
|
||||
[[ -f "${p}" && -f "${p}.gate" ]] && \
|
||||
difference_q="${difference_q}$(diff $p.gate $p)"
|
||||
done
|
||||
|
||||
# If no pom files were changed we can easily say no new dependencies were added
|
||||
if [ -z "${difference_q}" ]; then
|
||||
echo " * This patch does not change any dependencies."
|
||||
else
|
||||
# Else we need to manually build spark to determine what, if any, dependencies
|
||||
# were added into the Spark assembly jar
|
||||
${MVN_BIN} clean package dependency:build-classpath -DskipTests 2>/dev/null | \
|
||||
sed -n -e '/Building Spark Project Assembly/,$p' | \
|
||||
grep --context=1 -m 2 "Dependencies classpath:" | \
|
||||
head -n 3 | \
|
||||
tail -n 1 | \
|
||||
tr ":" "\n" | \
|
||||
rev | \
|
||||
cut -d "/" -f 1 | \
|
||||
rev | \
|
||||
sort > ${CURR_CP_FILE}
|
||||
|
||||
# Checkout the master branch to compare against
|
||||
git checkout master &>/dev/null
|
||||
|
||||
${MVN_BIN} clean package dependency:build-classpath -DskipTests 2>/dev/null | \
|
||||
sed -n -e '/Building Spark Project Assembly/,$p' | \
|
||||
grep --context=1 -m 2 "Dependencies classpath:" | \
|
||||
head -n 3 | \
|
||||
tail -n 1 | \
|
||||
tr ":" "\n" | \
|
||||
rev | \
|
||||
cut -d "/" -f 1 | \
|
||||
rev | \
|
||||
sort > ${MASTER_CP_FILE}
|
||||
|
||||
DIFF_RESULTS="`diff my-classpath.txt master-classpath.txt`"
|
||||
|
||||
if [ -z "${DIFF_RESULTS}" ]; then
|
||||
echo " * This patch does not change any dependencies."
|
||||
else
|
||||
# Pretty print the new dependencies
|
||||
added_deps=$(echo "${DIFF_RESULTS}" | grep "<" | cut -d' ' -f2 | awk '{print " * \`"$1"\`"}')
|
||||
removed_deps=$(echo "${DIFF_RESULTS}" | grep ">" | cut -d' ' -f2 | awk '{print " * \`"$1"\`"}')
|
||||
added_deps_text=" * This patch **adds the following new dependencies:**\n${added_deps}"
|
||||
removed_deps_text=" * This patch **removes the following dependencies:**\n${removed_deps}"
|
||||
|
||||
# Construct the final returned message with proper
|
||||
return_mssg=""
|
||||
[ -n "${added_deps}" ] && return_mssg="${added_deps_text}"
|
||||
if [ -n "${removed_deps}" ]; then
|
||||
if [ -n "${return_mssg}" ]; then
|
||||
return_mssg="${return_mssg}\n${removed_deps_text}"
|
||||
else
|
||||
return_mssg="${removed_deps_text}"
|
||||
fi
|
||||
fi
|
||||
echo "${return_mssg}"
|
||||
fi
|
||||
|
||||
# Remove the files we've left over
|
||||
[ -f "${CURR_CP_FILE}" ] && rm -f "${CURR_CP_FILE}"
|
||||
[ -f "${MASTER_CP_FILE}" ] && rm -f "${MASTER_CP_FILE}"
|
||||
|
||||
# Clean up our mess from the Maven builds just in case
|
||||
${MVN_BIN} clean &>/dev/null
|
||||
fi
|
||||
Loading…
Reference in a new issue