From 663cbdfbe5da6fa4af969344a6281c90b711207e Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Sat, 24 Jul 2021 16:49:11 +0900 Subject: [PATCH] [SPARK-36279][INFRA][PYTHON] Fix lint-python to work with Python 3.9 ### What changes were proposed in this pull request? Fix `lint-python` to pick `PYTHON_EXECUTABLE` from the environment variable first to switch the Python and explicitly specify `PYTHON_EXECUTABLE` to use `python3.9` in CI. ### Why are the changes needed? Currently `lint-python` uses `python3`, but it's not the one we expect in CI. As a result, `black` check is not working. ``` The python3 -m black command was not found. Skipping black checks for now. ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? The `black` check in `lint-python` should work. Closes #33507 from ueshin/issues/SPARK-36279/lint-python. Authored-by: Takuya UESHIN Signed-off-by: Hyukjin Kwon --- .github/workflows/build_and_test.yml | 2 +- dev/lint-python | 2 +- python/pyspark/pandas/tests/data_type_ops/test_num_ops.py | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 760be4adb6..39ddc2ce3c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -415,7 +415,7 @@ jobs: - name: Java linter run: ./dev/lint-java - name: Python linter - run: ./dev/lint-python + run: PYTHON_EXECUTABLE=python3.9 ./dev/lint-python - name: R linter run: ./dev/lint-r - name: JS linter diff --git a/dev/lint-python b/dev/lint-python index 8650058ed9..e54e391c58 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -24,7 +24,7 @@ MYPY_BUILD="mypy" PYCODESTYLE_BUILD="pycodestyle" MINIMUM_PYCODESTYLE="2.7.0" -PYTHON_EXECUTABLE="python3" +PYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-python3}" BLACK_BUILD="$PYTHON_EXECUTABLE -m black" diff --git a/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py b/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py index b6f965a1b7..fb2d2be78b 100644 --- a/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py +++ b/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py @@ -358,9 +358,7 @@ class NumOpsTest(PandasOnSparkTestCase, TestCasesUtils): for col in self.numeric_df_cols: pser, psser = pdf[col], psdf[col] if isinstance(psser.spark.data_type, DecimalType): - self.assertRaisesRegex( - TypeError, "< can not be applied to", lambda: psser < psser - ) + self.assertRaisesRegex(TypeError, "< can not be applied to", lambda: psser < psser) else: self.assert_eq(pser < pser, psser < psser) @@ -380,9 +378,7 @@ class NumOpsTest(PandasOnSparkTestCase, TestCasesUtils): for col in self.numeric_df_cols: pser, psser = pdf[col], psdf[col] if isinstance(psser.spark.data_type, DecimalType): - self.assertRaisesRegex( - TypeError, "> can not be applied to", lambda: psser > psser - ) + self.assertRaisesRegex(TypeError, "> can not be applied to", lambda: psser > psser) else: self.assert_eq(pser > pser, psser > psser)