7bbbe380c5
This commit parallelizes the Python unit test execution, significantly reducing Jenkins build times. Parallelism is now configurable by passing the `-p` or `--parallelism` flags to either `dev/run-tests` or `python/run-tests` (the default parallelism is 4, but I've successfully tested with higher parallelism). To avoid flakiness, I've disabled the Spark Web UI for the Python tests, similar to what we've done for the JVM tests. Author: Josh Rosen <joshrosen@databricks.com> Closes #7031 from JoshRosen/parallelize-python-tests and squashes the following commits: feb3763 [Josh Rosen] Re-enable other tests f87ea81 [Josh Rosen] Only log output from failed tests d4ded73 [Josh Rosen] Logging improvements a2717e1 [Josh Rosen] Make parallelism configurable via dev/run-tests 1bacf1b [Josh Rosen] Merge remote-tracking branch 'origin/master' into parallelize-python-tests 110cd9d [Josh Rosen] Fix universal_newlines for Python 3 cd13db8 [Josh Rosen] Also log python_implementation 9e31127 [Josh Rosen] Log Python --version output for each executable. a2b9094 [Josh Rosen] Bump up parallelism. 5552380 [Josh Rosen] Python 3 fix 866b5b9 [Josh Rosen] Fix lazy logging warnings in Prospector checks 87cb988 [Josh Rosen] Skip MLLib tests for PyPy 8309bfe [Josh Rosen] Temporarily disable parallelism to debug a failure 9129027 [Josh Rosen] Disable Spark UI in Python tests 037b686 [Josh Rosen] Temporarily disable JVM tests so we can test Python speedup in Jenkins. af4cef4 [Josh Rosen] Initial attempt at parallelizing Python test execution
83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
from __future__ import print_function
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def exit_from_command_with_retcode(cmd, retcode):
|
|
print("[error] running", ' '.join(cmd), "; received return code", retcode)
|
|
sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
|
|
|
|
|
|
def rm_r(path):
|
|
"""
|
|
Given an arbitrary path, properly remove it with the correct Python construct if it exists.
|
|
From: http://stackoverflow.com/a/9559881
|
|
"""
|
|
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path)
|
|
elif os.path.exists(path):
|
|
os.remove(path)
|
|
|
|
|
|
def run_cmd(cmd):
|
|
"""
|
|
Given a command as a list of arguments will attempt to execute the command
|
|
and, on failure, print an error message and exit.
|
|
"""
|
|
|
|
if not isinstance(cmd, list):
|
|
cmd = cmd.split()
|
|
try:
|
|
subprocess.check_call(cmd)
|
|
except subprocess.CalledProcessError as e:
|
|
exit_from_command_with_retcode(e.cmd, e.returncode)
|
|
|
|
|
|
def is_exe(path):
|
|
"""
|
|
Check if a given path is an executable file.
|
|
From: http://stackoverflow.com/a/377028
|
|
"""
|
|
|
|
return os.path.isfile(path) and os.access(path, os.X_OK)
|
|
|
|
|
|
def which(program):
|
|
"""
|
|
Find and return the given program by its absolute path or 'None' if the program cannot be found.
|
|
From: http://stackoverflow.com/a/377028
|
|
"""
|
|
|
|
fpath = os.path.split(program)[0]
|
|
|
|
if fpath:
|
|
if is_exe(program):
|
|
return program
|
|
else:
|
|
for path in os.environ.get("PATH").split(os.pathsep):
|
|
path = path.strip('"')
|
|
exe_file = os.path.join(path, program)
|
|
if is_exe(exe_file):
|
|
return exe_file
|
|
return None
|