d3180c25d8
This commit refactors the `run-tests-jenkins` script into Python. This refactoring was done by brennonyork in #7401; this PR contains a few minor edits from joshrosen in order to bring it up to date with other recent changes. From the original PR description (by brennonyork): Currently a few things are left out that, could and I think should, be smaller JIRA's after this. 1. There are still a few areas where we use environment variables where we don't need to (like `CURRENT_BLOCK`). I might get around to fixing this one in lieu of everything else, but wanted to point that out. 2. The PR tests are still written in bash. I opted to not change those and just rewrite the runner into Python. This is a great follow-on JIRA IMO. 3. All of the linting scripts are still in bash as well and would likely do to just add those in as follow-on JIRA's as well. Closes #7401. Author: Brennon York <brennon.york@capitalone.com> Closes #9161 from JoshRosen/run-tests-jenkins-refactoring.
116 lines
3.6 KiB
Python
116 lines
3.6 KiB
Python
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
from __future__ import print_function
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
if sys.version_info >= (2, 7):
|
|
subprocess_check_output = subprocess.check_output
|
|
subprocess_check_call = subprocess.check_call
|
|
else:
|
|
# SPARK-8763
|
|
# backported from subprocess module in Python 2.7
|
|
def subprocess_check_output(*popenargs, **kwargs):
|
|
if 'stdout' in kwargs:
|
|
raise ValueError('stdout argument not allowed, it will be overridden.')
|
|
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
|
|
output, unused_err = process.communicate()
|
|
retcode = process.poll()
|
|
if retcode:
|
|
cmd = kwargs.get("args")
|
|
if cmd is None:
|
|
cmd = popenargs[0]
|
|
raise subprocess.CalledProcessError(retcode, cmd, output=output)
|
|
return output
|
|
|
|
# backported from subprocess module in Python 2.7
|
|
def subprocess_check_call(*popenargs, **kwargs):
|
|
retcode = call(*popenargs, **kwargs)
|
|
if retcode:
|
|
cmd = kwargs.get("args")
|
|
if cmd is None:
|
|
cmd = popenargs[0]
|
|
raise CalledProcessError(retcode, cmd)
|
|
return 0
|
|
|
|
|
|
def exit_from_command_with_retcode(cmd, retcode):
|
|
print("[error] running", ' '.join(cmd), "; received return code", retcode)
|
|
sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
|
|
|
|
|
|
def rm_r(path):
|
|
"""
|
|
Given an arbitrary path, properly remove it with the correct Python construct if it exists.
|
|
From: http://stackoverflow.com/a/9559881
|
|
"""
|
|
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path)
|
|
elif os.path.exists(path):
|
|
os.remove(path)
|
|
|
|
|
|
def run_cmd(cmd, return_output=False):
|
|
"""
|
|
Given a command as a list of arguments will attempt to execute the command
|
|
and, on failure, print an error message and exit.
|
|
"""
|
|
|
|
if not isinstance(cmd, list):
|
|
cmd = cmd.split()
|
|
try:
|
|
if return_output:
|
|
return subprocess_check_output(cmd)
|
|
else:
|
|
return subprocess_check_call(cmd)
|
|
except subprocess.CalledProcessError as e:
|
|
exit_from_command_with_retcode(e.cmd, e.returncode)
|
|
|
|
|
|
def is_exe(path):
|
|
"""
|
|
Check if a given path is an executable file.
|
|
From: http://stackoverflow.com/a/377028
|
|
"""
|
|
|
|
return os.path.isfile(path) and os.access(path, os.X_OK)
|
|
|
|
|
|
def which(program):
|
|
"""
|
|
Find and return the given program by its absolute path or 'None' if the program cannot be found.
|
|
From: http://stackoverflow.com/a/377028
|
|
"""
|
|
|
|
fpath = os.path.split(program)[0]
|
|
|
|
if fpath:
|
|
if is_exe(program):
|
|
return program
|
|
else:
|
|
for path in os.environ.get("PATH").split(os.pathsep):
|
|
path = path.strip('"')
|
|
exe_file = os.path.join(path, program)
|
|
if is_exe(exe_file):
|
|
return exe_file
|
|
return None
|