ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-5161] [HOTFIX] Fix bug in Python test failure reporting

Browse source 
This patch fixes a bug introduced in #7031 which can cause Jenkins to incorrectly report a build with failed Python tests as passing if an error occurred while printing the test failure message.

Author: Josh Rosen <joshrosen@databricks.com>

Closes #7112 from JoshRosen/python-tests-hotfix and squashes the following commits:

c3f2961 [Josh Rosen] Hotfix for bug in Python test failure reporting
This commit is contained in:
Josh Rosen 2015-06-29 23:08:51 -07:00
parent e6c3f7462b
commit 6c5a6db4d5
1 changed files with 23 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
python/run-tests.py
View file

@ -58,22 +58,33 @@ def run_individual_python_test(test_name, pyspark_python):
env = {'SPARK_TESTING': '1', 'PYSPARK_PYTHON': which(pyspark_python)} env = {'SPARK_TESTING': '1', 'PYSPARK_PYTHON': which(pyspark_python)}
LOGGER.debug("Starting test(%s): %s", pyspark_python, test_name) LOGGER.debug("Starting test(%s): %s", pyspark_python, test_name)
start_time = time.time() start_time = time.time()
per_test_output = tempfile.TemporaryFile() try:
retcode = subprocess.Popen( per_test_output = tempfile.TemporaryFile()
[os.path.join(SPARK_HOME, "bin/pyspark"), test_name], retcode = subprocess.Popen(
stderr=per_test_output, stdout=per_test_output, env=env).wait() [os.path.join(SPARK_HOME, "bin/pyspark"), test_name],
stderr=per_test_output, stdout=per_test_output, env=env).wait()
except:
LOGGER.exception("Got exception while running %s with %s", test_name, pyspark_python)
# Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
# this code is invoked from a thread other than the main thread.
os._exit(1)
duration = time.time() - start_time duration = time.time() - start_time
# Exit on the first failure. # Exit on the first failure.
if retcode != 0: if retcode != 0:
with FAILURE_REPORTING_LOCK: try:
with open(LOG_FILE, 'ab') as log_file: with FAILURE_REPORTING_LOCK:
with open(LOG_FILE, 'ab') as log_file:
per_test_output.seek(0)
log_file.writelines(per_test_output)
per_test_output.seek(0) per_test_output.seek(0)
log_file.writelines(per_test_output.readlines()) for line in per_test_output:
per_test_output.seek(0) decoded_line = line.decode()
for line in per_test_output: if not re.match('[0-9]+', decoded_line):
if not re.match('[0-9]+', line): print(decoded_line, end='')
print(line, end='') per_test_output.close()
per_test_output.close() except:
LOGGER.exception("Got an exception while trying to print failed test output")
finally:
print_red("\nHad test failures in %s with %s; see logs." % (test_name, pyspark_python)) print_red("\nHad test failures in %s with %s; see logs." % (test_name, pyspark_python))
# Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
# this code is invoked from a thread other than the main thread. # this code is invoked from a thread other than the main thread.