a29c9bd614
This PR sends archived unit tests logs to the build history directory in Jenkins master, so that we can serve it via HTTP later to help debugging Jenkins build failures. pwendell JoshRosen Please help review, thanks! Author: Cheng Lian <lian@databricks.com> Closes #2845 from liancheng/log-archive and squashes the following commits: ac8d9d4 [Cheng Lian] Includes build number in messages posted to GitHub 68c7010 [Cheng Lian] Logs backup should be implemented in dev/run-tests-jenkins 4b912f7 [Cheng Lian] Sends archived unit tests logs to Jenkins master
238 lines
8.5 KiB
Bash
Executable file
238 lines
8.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Wrapper script that runs the Spark tests then reports QA results
|
|
# to github via its API.
|
|
# Environment variables are populated by the code here:
|
|
#+ https://github.com/jenkinsci/ghprb-plugin/blob/master/src/main/java/org/jenkinsci/plugins/ghprb/GhprbTrigger.java#L139
|
|
|
|
# Go to the Spark project root directory
|
|
FWDIR="$(cd `dirname $0`/..; pwd)"
|
|
cd "$FWDIR"
|
|
|
|
source "$FWDIR/dev/run-tests-codes.sh"
|
|
|
|
COMMENTS_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments"
|
|
PULL_REQUEST_URL="https://github.com/apache/spark/pull/$ghprbPullId"
|
|
|
|
# Important Environment Variables
|
|
# ---
|
|
# $ghprbActualCommit
|
|
#+ This is the hash of the most recent commit in the PR.
|
|
#+ The merge-base of this and master is the commit from which the PR was branched.
|
|
# $sha1
|
|
#+ If the patch merges cleanly, this is a reference to the merge commit hash
|
|
#+ (e.g. "origin/pr/2606/merge").
|
|
#+ If the patch does not merge cleanly, it is equal to $ghprbActualCommit.
|
|
#+ The merge-base of this and master in the case of a clean merge is the most recent commit
|
|
#+ against master.
|
|
|
|
COMMIT_URL="https://github.com/apache/spark/commit/${ghprbActualCommit}"
|
|
# GitHub doesn't auto-link short hashes when submitted via the API, unfortunately. :(
|
|
SHORT_COMMIT_HASH="${ghprbActualCommit:0:7}"
|
|
|
|
TESTS_TIMEOUT="120m" # format: http://linux.die.net/man/1/timeout
|
|
|
|
function post_message () {
|
|
local message=$1
|
|
local data="{\"body\": \"$message\"}"
|
|
local HTTP_CODE_HEADER="HTTP Response Code: "
|
|
|
|
echo "Attempting to post to Github..."
|
|
|
|
local curl_output=$(
|
|
curl `#--dump-header -` \
|
|
--silent \
|
|
--user x-oauth-basic:$GITHUB_OAUTH_KEY \
|
|
--request POST \
|
|
--data "$data" \
|
|
--write-out "${HTTP_CODE_HEADER}%{http_code}\n" \
|
|
--header "Content-Type: application/json" \
|
|
"$COMMENTS_URL" #> /dev/null #| "$FWDIR/dev/jq" .id #| head -n 8
|
|
)
|
|
local curl_status=${PIPESTATUS[0]}
|
|
|
|
if [ "$curl_status" -ne 0 ]; then
|
|
echo "Failed to post message to GitHub." >&2
|
|
echo " > curl_status: ${curl_status}" >&2
|
|
echo " > curl_output: ${curl_output}" >&2
|
|
echo " > data: ${data}" >&2
|
|
# exit $curl_status
|
|
fi
|
|
|
|
local api_response=$(
|
|
echo "${curl_output}" \
|
|
| grep -v -e "^${HTTP_CODE_HEADER}"
|
|
)
|
|
|
|
local http_code=$(
|
|
echo "${curl_output}" \
|
|
| grep -e "^${HTTP_CODE_HEADER}" \
|
|
| sed -r -e "s/^${HTTP_CODE_HEADER}//g"
|
|
)
|
|
|
|
if [ -n "$http_code" ] && [ "$http_code" -ne "201" ]; then
|
|
echo " > http_code: ${http_code}." >&2
|
|
echo " > api_response: ${api_response}" >&2
|
|
echo " > data: ${data}" >&2
|
|
fi
|
|
|
|
if [ "$curl_status" -eq 0 ] && [ "$http_code" -eq "201" ]; then
|
|
echo " > Post successful."
|
|
fi
|
|
}
|
|
|
|
function send_archived_logs () {
|
|
echo "Archiving unit tests logs..."
|
|
|
|
local log_files=$(find . -name "unit-tests.log")
|
|
|
|
if [ -z "$log_files" ]; then
|
|
echo "> No log files found." >&2
|
|
else
|
|
local log_archive="unit-tests-logs.tar.gz"
|
|
echo "$log_files" | xargs tar czf ${log_archive}
|
|
|
|
local jenkins_build_dir=${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}
|
|
local scp_output=$(scp ${log_archive} amp-jenkins-master:${jenkins_build_dir}/${log_archive})
|
|
local scp_status="$?"
|
|
|
|
if [ "$scp_status" -ne 0 ]; then
|
|
echo "Failed to send archived unit tests logs to Jenkins master." >&2
|
|
echo "> scp_status: ${scp_status}" >&2
|
|
echo "> scp_output: ${scp_output}" >&2
|
|
else
|
|
echo "> Send successful."
|
|
fi
|
|
|
|
rm -f ${log_archive}
|
|
fi
|
|
}
|
|
|
|
|
|
# We diff master...$ghprbActualCommit because that gets us changes introduced in the PR
|
|
#+ and not anything else added to master since the PR was branched.
|
|
|
|
# check PR merge-ability and check for new public classes
|
|
{
|
|
if [ "$sha1" == "$ghprbActualCommit" ]; then
|
|
merge_note=" * This patch **does not merge cleanly**."
|
|
else
|
|
merge_note=" * This patch merges cleanly."
|
|
fi
|
|
|
|
source_files=$(
|
|
git diff master...$ghprbActualCommit --name-only `# diff patch against master from branch point` \
|
|
| grep -v -e "\/test" `# ignore files in test directories` \
|
|
| grep -e "\.py$" -e "\.java$" -e "\.scala$" `# include only code files` \
|
|
| tr "\n" " "
|
|
)
|
|
new_public_classes=$(
|
|
git diff master...$ghprbActualCommit ${source_files} `# diff patch against master from branch point` \
|
|
| grep "^\+" `# filter in only added lines` \
|
|
| sed -r -e "s/^\+//g" `# remove the leading +` \
|
|
| grep -e "trait " -e "class " `# filter in lines with these key words` \
|
|
| grep -e "{" -e "(" `# filter in lines with these key words, too` \
|
|
| grep -v -e "\@\@" -e "private" `# exclude lines with these words` \
|
|
| grep -v -e "^// " -e "^/\*" -e "^ \* " `# exclude comment lines` \
|
|
| sed -r -e "s/\{.*//g" `# remove from the { onwards` \
|
|
| sed -r -e "s/\}//g" `# just in case, remove }; they mess the JSON` \
|
|
| sed -r -e "s/\"/\\\\\"/g" `# escape double quotes; they mess the JSON` \
|
|
| sed -r -e "s/^(.*)$/\`\1\`/g" `# surround with backticks for style` \
|
|
| sed -r -e "s/^/ \* /g" `# prepend ' *' to start of line` \
|
|
| sed -r -e "s/$/\\\n/g" `# append newline to end of line` \
|
|
| tr -d "\n" `# remove actual LF characters`
|
|
)
|
|
|
|
if [ -z "$new_public_classes" ]; then
|
|
public_classes_note=" * This patch adds no public classes."
|
|
else
|
|
public_classes_note=" * This patch adds the following public classes _(experimental)_:"
|
|
public_classes_note="${public_classes_note}\n${new_public_classes}"
|
|
fi
|
|
}
|
|
|
|
# post start message
|
|
{
|
|
start_message="\
|
|
[Test build ${BUILD_DISPLAY_NAME} has started](${BUILD_URL}consoleFull) for \
|
|
PR $ghprbPullId at commit [\`${SHORT_COMMIT_HASH}\`](${COMMIT_URL})."
|
|
|
|
start_message="${start_message}\n${merge_note}"
|
|
# start_message="${start_message}\n${public_classes_note}"
|
|
|
|
post_message "$start_message"
|
|
}
|
|
|
|
# run tests
|
|
{
|
|
timeout "${TESTS_TIMEOUT}" ./dev/run-tests
|
|
test_result="$?"
|
|
|
|
if [ "$test_result" -eq "124" ]; then
|
|
fail_message="**[Test build ${BUILD_DISPLAY_NAME} timed out](${BUILD_URL}consoleFull)** \
|
|
for PR $ghprbPullId at commit [\`${SHORT_COMMIT_HASH}\`](${COMMIT_URL}) \
|
|
after a configured wait of \`${TESTS_TIMEOUT}\`."
|
|
|
|
post_message "$fail_message"
|
|
exit $test_result
|
|
elif [ "$test_result" -eq "0" ]; then
|
|
test_result_note=" * This patch **passes all tests**."
|
|
else
|
|
if [ "$test_result" -eq "$BLOCK_GENERAL" ]; then
|
|
failing_test="some tests"
|
|
elif [ "$test_result" -eq "$BLOCK_RAT" ]; then
|
|
failing_test="RAT tests"
|
|
elif [ "$test_result" -eq "$BLOCK_SCALA_STYLE" ]; then
|
|
failing_test="Scala style tests"
|
|
elif [ "$test_result" -eq "$BLOCK_PYTHON_STYLE" ]; then
|
|
failing_test="Python style tests"
|
|
elif [ "$test_result" -eq "$BLOCK_BUILD" ]; then
|
|
failing_test="to build"
|
|
elif [ "$test_result" -eq "$BLOCK_SPARK_UNIT_TESTS" ]; then
|
|
failing_test="Spark unit tests"
|
|
elif [ "$test_result" -eq "$BLOCK_PYSPARK_UNIT_TESTS" ]; then
|
|
failing_test="PySpark unit tests"
|
|
elif [ "$test_result" -eq "$BLOCK_MIMA" ]; then
|
|
failing_test="MiMa tests"
|
|
else
|
|
failing_test="some tests"
|
|
fi
|
|
|
|
test_result_note=" * This patch **fails $failing_test**."
|
|
fi
|
|
|
|
send_archived_logs
|
|
}
|
|
|
|
# post end message
|
|
{
|
|
result_message="\
|
|
[Test build ${BUILD_DISPLAY_NAME} has finished](${BUILD_URL}consoleFull) for \
|
|
PR $ghprbPullId at commit [\`${SHORT_COMMIT_HASH}\`](${COMMIT_URL})."
|
|
|
|
result_message="${result_message}\n${test_result_note}"
|
|
result_message="${result_message}\n${merge_note}"
|
|
result_message="${result_message}\n${public_classes_note}"
|
|
|
|
post_message "$result_message"
|
|
}
|
|
|
|
exit $test_result
|