spark-instrumented-optimizer/dev/lint-python

#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# define test binaries + versions
FLAKE8_BUILD="flake8"
MINIMUM_FLAKE8="3.5.0"
MYPY_BUILD="mypy"
PYCODESTYLE_BUILD="pycodestyle"
MINIMUM_PYCODESTYLE="2.6.0"

SPHINX_BUILD="sphinx-build"

PYTHON_EXECUTABLE="python3"

function satisfies_min_version {
    local provided_version="$1"
    local expected_version="$2"
    echo "$(
        "$PYTHON_EXECUTABLE" << EOM
from setuptools.extern.packaging import version
print(version.parse('$provided_version') >= version.parse('$expected_version'))
EOM
    )"
}

function compile_python_test {
    local COMPILE_STATUS=
    local COMPILE_REPORT=

    if [[ ! "$1" ]]; then
        echo "No python files found!  Something is very wrong -- exiting."
        exit 1;
    fi

    # compileall: https://docs.python.org/3/library/compileall.html
    echo "starting python compilation test..."
    COMPILE_REPORT=$( ("$PYTHON_EXECUTABLE" -B -mcompileall -q -l -x "[/\\\\][.]git" $1) 2>&1)
    COMPILE_STATUS=$?

    if [ $COMPILE_STATUS -ne 0 ]; then
        echo "Python compilation failed with the following errors:"
        echo "$COMPILE_REPORT"
        echo "$COMPILE_STATUS"
        exit "$COMPILE_STATUS"
    else
        echo "python compilation succeeded."
        echo
    fi
}

function pycodestyle_test {
    local PYCODESTYLE_STATUS=
    local PYCODESTYLE_REPORT=
    local RUN_LOCAL_PYCODESTYLE=
    local PYCODESTYLE_VERSION=
    local EXPECTED_PYCODESTYLE=
    local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
    local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"

    if [[ ! "$1" ]]; then
        echo "No python files found!  Something is very wrong -- exiting."
        exit 1;
    fi

    # check for locally installed pycodestyle & version
    RUN_LOCAL_PYCODESTYLE="False"
    if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
        PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)"
        EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)"
        if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
            RUN_LOCAL_PYCODESTYLE="True"
        fi
    fi

    # download the right version or run locally
    if [ $RUN_LOCAL_PYCODESTYLE == "False" ]; then
        # Get pycodestyle at runtime so that we don't rely on it being installed on the build server.
        # See: https://github.com/apache/spark/pull/1744#issuecomment-50982162
        # Updated to the latest official version of pep8. pep8 is formally renamed to pycodestyle.
        echo "downloading pycodestyle from $PYCODESTYLE_SCRIPT_REMOTE_PATH..."
        if [ ! -e "$PYCODESTYLE_SCRIPT_PATH" ]; then
            curl --silent -o "$PYCODESTYLE_SCRIPT_PATH" "$PYCODESTYLE_SCRIPT_REMOTE_PATH"
            local curl_status="$?"

            if [ "$curl_status" -ne 0 ]; then
                echo "Failed to download pycodestyle.py from $PYCODESTYLE_SCRIPT_REMOTE_PATH"
                exit "$curl_status"
            fi
        fi

        echo "starting pycodestyle test..."
        PYCODESTYLE_REPORT=$( ("$PYTHON_EXECUTABLE" "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $1) 2>&1)
        PYCODESTYLE_STATUS=$?
    else
        # we have the right version installed, so run locally
        echo "starting pycodestyle test..."
        PYCODESTYLE_REPORT=$( ($PYCODESTYLE_BUILD --config=dev/tox.ini $1) 2>&1)
        PYCODESTYLE_STATUS=$?
    fi

    if [ $PYCODESTYLE_STATUS -ne 0 ]; then
        echo "pycodestyle checks failed:"
        echo "$PYCODESTYLE_REPORT"
        exit "$PYCODESTYLE_STATUS"
    else
        echo "pycodestyle checks passed."
        echo
    fi
}

function mypy_test {
    local MYPY_REPORT=
    local MYPY_STATUS=

    # TODO(SPARK-32797): Install mypy on the Jenkins CI workers
    if ! hash "$MYPY_BUILD" 2> /dev/null; then
        echo "The $MYPY_BUILD command was not found. Skipping for now."
        return
    fi

    echo "starting $MYPY_BUILD test..."
    MYPY_REPORT=$( ($MYPY_BUILD --config-file python/mypy.ini python/pyspark) 2>&1)
    MYPY_STATUS=$?

    if [ "$MYPY_STATUS" -ne 0 ]; then
        echo "mypy checks failed:"
        echo "$MYPY_REPORT"
        echo "$MYPY_STATUS"
        exit "$MYPY_STATUS"
    else
        echo "mypy checks passed."
        echo
    fi
}

function flake8_test {
    local FLAKE8_VERSION=
    local EXPECTED_FLAKE8=
    local FLAKE8_REPORT=
    local FLAKE8_STATUS=

    if ! hash "$FLAKE8_BUILD" 2> /dev/null; then
        echo "The flake8 command was not found."
        echo "flake8 checks failed."
        exit 1
    fi

    _FLAKE8_VERSION=($($FLAKE8_BUILD --version))
    FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
    EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"

    if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
        echo "\
The minimum flake8 version needs to be $MINIMUM_FLAKE8. Your current version is $FLAKE8_VERSION

flake8 checks failed."
        exit 1
    fi

    echo "starting $FLAKE8_BUILD test..."
    FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1)
    FLAKE8_STATUS=$?

    if [ "$FLAKE8_STATUS" -ne 0 ]; then
        echo "flake8 checks failed:"
        echo "$FLAKE8_REPORT"
        echo "$FLAKE8_STATUS"
        exit "$FLAKE8_STATUS"
    else
        echo "flake8 checks passed."
        echo
    fi
}

function sphinx_test {
    local SPHINX_REPORT=
    local SPHINX_STATUS=

    # Check that the documentation builds acceptably, skip check if sphinx is not installed.
    if ! hash "$SPHINX_BUILD" 2> /dev/null; then
        echo "The $SPHINX_BUILD command was not found. Skipping Sphinx build for now."
        echo
        return
    fi

    PYTHON_HAS_SPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("sphinx") is not None)')
    if [[ "$PYTHON_HAS_SPHINX" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE does not have Sphinx installed. Skipping Sphinx build for now."
        echo
        return
    fi

    # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
    #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
    PYTHON_HAS_SPHINX_3_0=$("$PYTHON_EXECUTABLE" -c 'from distutils.version import LooseVersion; import sphinx; print(LooseVersion(sphinx.__version__) < LooseVersion("3.1.0"))')
    if [[ "$PYTHON_HAS_SPHINX_3_0" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE has Sphinx 3.1+ installed but it requires lower than 3.1. Skipping Sphinx build for now."
        echo
        return
    fi

    # TODO(SPARK-32391): Install pydata_sphinx_theme in Jenkins machines
    PYTHON_HAS_THEME=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("pydata_sphinx_theme") is not None)')
    if [[ "$PYTHON_HAS_THEME" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE does not have pydata_sphinx_theme installed. Skipping Sphinx build for now."
        echo
        return
    fi

    # TODO(SPARK-32666): Install nbsphinx in Jenkins machines
    PYTHON_HAS_NBSPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("nbsphinx") is not None)')
    if [[ "$PYTHON_HAS_NBSPHINX" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE does not have nbsphinx installed. Skipping Sphinx build for now."
        echo
        return
    fi

    # TODO(SPARK-32666): Install ipython in Jenkins machines
    PYTHON_HAS_IPYTHON=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("IPython") is not None)')
    if [[ "$PYTHON_HAS_IPYTHON" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE does not have ipython installed. Skipping Sphinx build for now."
        echo
        return
    fi

    # TODO(SPARK-33242): Install numpydoc in Jenkins machines
    PYTHON_HAS_NUMPYDOC=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("numpydoc") is not None)')
    if [[ "$PYTHON_HAS_NUMPYDOC" == "False" ]]; then
        echo "$PYTHON_EXECUTABLE does not have numpydoc installed. Skipping Sphinx build for now."
        echo
        return
    fi

    echo "starting $SPHINX_BUILD tests..."
    pushd python/docs &> /dev/null
    make clean &> /dev/null
    # Treat warnings as errors so we stop correctly
    SPHINX_REPORT=$( (SPHINXOPTS="-a -W" make html) 2>&1)
    SPHINX_STATUS=$?

    if [ "$SPHINX_STATUS" -ne 0 ]; then
        echo "$SPHINX_BUILD checks failed:"
        echo "$SPHINX_REPORT"
        echo
        echo "re-running make html to print full warning list:"
        make clean &> /dev/null
        SPHINX_REPORT=$( (SPHINXOPTS="-a" make html) 2>&1)
        echo "$SPHINX_REPORT"
        exit "$SPHINX_STATUS"
    else
        echo "$SPHINX_BUILD checks passed."
        echo
    fi

    popd &> /dev/null
}

SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
SPARK_ROOT_DIR="$(dirname "${SCRIPT_DIR}")"

pushd "$SPARK_ROOT_DIR" &> /dev/null

PYTHON_SOURCE="$(find . -name "*.py")"

compile_python_test "$PYTHON_SOURCE"
pycodestyle_test "$PYTHON_SOURCE"
flake8_test
mypy_test
sphinx_test

echo
echo "all lint-python tests passed!"

popd &> /dev/null