[SPARK-23174][BUILD][PYTHON] python code style checker update
## What changes were proposed in this pull request? Referencing latest python code style checking from PyPi/pycodestyle Removed pending TODO For now, in tox.ini excluded the additional style error discovered on existing python due to latest style checker (will fallback on review comment to finalize exclusion or fix py) Any further code styling requirement needs to be part of pycodestyle, not in SPARK. ## How was this patch tested? ./dev/run-tests Author: Rekha Joshi <rekhajoshm@gmail.com> Author: rjoshi2 <rekhajoshm@gmail.com> Closes #20338 from rekhajoshm/SPARK-11222.
This commit is contained in:
parent
4e7b49041a
commit
7af1a325da
|
@ -21,7 +21,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
|
|||
SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
# Exclude auto-generated configuration file.
|
||||
PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )"
|
||||
PEP8_REPORT_PATH="$SPARK_ROOT_DIR/dev/pep8-report.txt"
|
||||
PYCODESTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-report.txt"
|
||||
PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt"
|
||||
PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt"
|
||||
SPHINXBUILD=${SPHINXBUILD:=sphinx-build}
|
||||
|
@ -30,23 +30,22 @@ SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt"
|
|||
cd "$SPARK_ROOT_DIR"
|
||||
|
||||
# compileall: https://docs.python.org/2/library/compileall.html
|
||||
python -B -m compileall -q -l $PATHS_TO_CHECK > "$PEP8_REPORT_PATH"
|
||||
python -B -m compileall -q -l $PATHS_TO_CHECK > "$PYCODESTYLE_REPORT_PATH"
|
||||
compile_status="${PIPESTATUS[0]}"
|
||||
|
||||
# Get pep8 at runtime so that we don't rely on it being installed on the build server.
|
||||
# Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
|
||||
#+ See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
|
||||
#+ TODOs:
|
||||
#+ - Download pep8 from PyPI. It's more "official".
|
||||
PEP8_VERSION="1.7.0"
|
||||
PEP8_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pep8-$PEP8_VERSION.py"
|
||||
PEP8_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/jcrocholl/pep8/$PEP8_VERSION/pep8.py"
|
||||
# Updated to latest official version for pep8. pep8 is formally renamed to pycodestyle.
|
||||
PYCODESTYLE_VERSION="2.3.1"
|
||||
PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$PYCODESTYLE_VERSION.py"
|
||||
PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$PYCODESTYLE_VERSION/pycodestyle.py"
|
||||
|
||||
if [ ! -e "$PEP8_SCRIPT_PATH" ]; then
|
||||
curl --silent -o "$PEP8_SCRIPT_PATH" "$PEP8_SCRIPT_REMOTE_PATH"
|
||||
if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
|
||||
curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
|
||||
curl_status="$?"
|
||||
|
||||
if [ "$curl_status" -ne 0 ]; then
|
||||
echo "Failed to download pep8.py from \"$PEP8_SCRIPT_REMOTE_PATH\"."
|
||||
echo "Failed to download pycodestyle.py from \"$PYCODESTYLE_SCRIPT_REMOTE_PATH\"."
|
||||
exit "$curl_status"
|
||||
fi
|
||||
fi
|
||||
|
@ -64,23 +63,23 @@ export "PATH=$PYTHONPATH:$PATH"
|
|||
#+ first, but we do so so that the check status can
|
||||
#+ be output before the report, like with the
|
||||
#+ scalastyle and RAT checks.
|
||||
python "$PEP8_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH"
|
||||
pep8_status="${PIPESTATUS[0]}"
|
||||
python "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PYCODESTYLE_REPORT_PATH"
|
||||
pycodestyle_status="${PIPESTATUS[0]}"
|
||||
|
||||
if [ "$compile_status" -eq 0 -a "$pep8_status" -eq 0 ]; then
|
||||
if [ "$compile_status" -eq 0 -a "$pycodestyle_status" -eq 0 ]; then
|
||||
lint_status=0
|
||||
else
|
||||
lint_status=1
|
||||
fi
|
||||
|
||||
if [ "$lint_status" -ne 0 ]; then
|
||||
echo "PEP8 checks failed."
|
||||
cat "$PEP8_REPORT_PATH"
|
||||
rm "$PEP8_REPORT_PATH"
|
||||
echo "PYCODESTYLE checks failed."
|
||||
cat "$PYCODESTYLE_REPORT_PATH"
|
||||
rm "$PYCODESTYLE_REPORT_PATH"
|
||||
exit "$lint_status"
|
||||
else
|
||||
echo "PEP8 checks passed."
|
||||
rm "$PEP8_REPORT_PATH"
|
||||
echo "pycodestyle checks passed."
|
||||
rm "$PYCODESTYLE_REPORT_PATH"
|
||||
fi
|
||||
|
||||
# Check that the documentation builds acceptably, skip check if sphinx is not installed.
|
||||
|
|
|
@ -576,7 +576,10 @@ def main():
|
|||
for f in changed_files):
|
||||
# run_java_style_checks()
|
||||
pass
|
||||
if not changed_files or any(f.endswith(".py") for f in changed_files):
|
||||
if not changed_files or any(f.endswith("lint-python")
|
||||
or f.endswith("tox.ini")
|
||||
or f.endswith(".py")
|
||||
for f in changed_files):
|
||||
run_python_style_checks()
|
||||
if not changed_files or any(f.endswith(".R")
|
||||
or f.endswith("lint-r")
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[pep8]
|
||||
ignore=E402,E731,E241,W503,E226
|
||||
[pycodestyle]
|
||||
ignore=E402,E731,E241,W503,E226,E722,E741,E305
|
||||
max-line-length=100
|
||||
exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*
|
||||
|
|
Loading…
Reference in a new issue