[SPARK-8763] [PYSPARK] executing run-tests.py with Python 2.6 fails with absence of subprocess.check_output function
Running run-tests.py with Python 2.6 cause following error: ``` Running PySpark tests. Output is in python//Users/tomohiko/.jenkins/jobs/pyspark_test/workspace/python/unit-tests.log Will test against the following Python executables: ['python2.6', 'python3.4', 'pypy'] Will test the following Python modules: ['pyspark-core', 'pyspark-ml', 'pyspark-mllib', 'pyspark-sql', 'pyspark-streaming'] Traceback (most recent call last): File "./python/run-tests.py", line 196, in <module> main() File "./python/run-tests.py", line 159, in main python_implementation = subprocess.check_output( AttributeError: 'module' object has no attribute 'check_output' ... ``` The cause of this error is using subprocess.check_output function, which exists since Python 2.7. (ref. https://docs.python.org/2.7/library/subprocess.html#subprocess.check_output) Author: cocoatomo <cocoatomo77@gmail.com> Closes #7161 from cocoatomo/issues/8763-test-fails-py26 and squashes the following commits: cf4f901 [cocoatomo] [SPARK-8763] backport process.check_output function from Python 2.7
This commit is contained in:
parent
97652416e2
commit
fdcad6ef48
|
@ -31,6 +31,23 @@ if sys.version < '3':
|
|||
import Queue
|
||||
else:
|
||||
import queue as Queue
|
||||
if sys.version_info >= (2, 7):
|
||||
subprocess_check_output = subprocess.check_output
|
||||
else:
|
||||
# SPARK-8763
|
||||
# backported from subprocess module in Python 2.7
|
||||
def subprocess_check_output(*popenargs, **kwargs):
|
||||
if 'stdout' in kwargs:
|
||||
raise ValueError('stdout argument not allowed, it will be overridden.')
|
||||
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
|
||||
output, unused_err = process.communicate()
|
||||
retcode = process.poll()
|
||||
if retcode:
|
||||
cmd = kwargs.get("args")
|
||||
if cmd is None:
|
||||
cmd = popenargs[0]
|
||||
raise subprocess.CalledProcessError(retcode, cmd, output=output)
|
||||
return output
|
||||
|
||||
|
||||
# Append `SPARK_HOME/dev` to the Python path so that we can import the sparktestsupport module
|
||||
|
@ -156,11 +173,11 @@ def main():
|
|||
|
||||
task_queue = Queue.Queue()
|
||||
for python_exec in python_execs:
|
||||
python_implementation = subprocess.check_output(
|
||||
python_implementation = subprocess_check_output(
|
||||
[python_exec, "-c", "import platform; print(platform.python_implementation())"],
|
||||
universal_newlines=True).strip()
|
||||
LOGGER.debug("%s python_implementation is %s", python_exec, python_implementation)
|
||||
LOGGER.debug("%s version is: %s", python_exec, subprocess.check_output(
|
||||
LOGGER.debug("%s version is: %s", python_exec, subprocess_check_output(
|
||||
[python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
|
||||
for module in modules_to_test:
|
||||
if python_implementation not in module.blacklisted_python_implementations:
|
||||
|
|
Loading…
Reference in a new issue