2020-09-24 01:15:36 -04:00
|
|
|
;
|
|
|
|
; Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
; contributor license agreements. See the NOTICE file distributed with
|
|
|
|
; this work for additional information regarding copyright ownership.
|
|
|
|
; The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
; (the "License"); you may not use this file except in compliance with
|
|
|
|
; the License. You may obtain a copy of the License at
|
|
|
|
;
|
|
|
|
; http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
;
|
|
|
|
; Unless required by applicable law or agreed to in writing, software
|
|
|
|
; distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
; See the License for the specific language governing permissions and
|
|
|
|
; limitations under the License.
|
|
|
|
;
|
|
|
|
|
|
|
|
[mypy]
|
2020-11-24 19:27:04 -05:00
|
|
|
strict_optional = True
|
|
|
|
no_implicit_optional = True
|
|
|
|
disallow_untyped_defs = True
|
|
|
|
|
|
|
|
; Allow untyped def in internal modules and tests
|
|
|
|
|
|
|
|
[mypy-pyspark.daemon]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.find_spark_home]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark._globals]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.install]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.java_gateway]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.join]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.ml.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.mllib.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.rddsampler]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.resource.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.serializers]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.shuffle]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.streaming.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.streaming.util]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.pandas.serializers]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.pandas.types]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.pandas.typehints]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.pandas.utils]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.pandas._typing.protocols.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.sql.utils]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.tests.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.testing.*]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.traceback_utils]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.util]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
[mypy-pyspark.worker]
|
|
|
|
disallow_untyped_defs = False
|
|
|
|
|
|
|
|
; Ignore errors in embedded third party code
|
2020-09-24 01:15:36 -04:00
|
|
|
|
[SPARK-32320][PYSPARK] Remove mutable default arguments
This is bad practice, and might lead to unexpected behaviour:
https://florimond.dev/blog/articles/2018/08/python-mutable-defaults-are-the-source-of-all-evil/
```
fokkodriesprongFan spark % grep -R "={}" python | grep def
python/pyspark/resource/profile.py: def __init__(self, _java_resource_profile=None, _exec_req={}, _task_req={}):
python/pyspark/sql/functions.py:def from_json(col, schema, options={}):
python/pyspark/sql/functions.py:def to_json(col, options={}):
python/pyspark/sql/functions.py:def schema_of_json(json, options={}):
python/pyspark/sql/functions.py:def schema_of_csv(csv, options={}):
python/pyspark/sql/functions.py:def to_csv(col, options={}):
python/pyspark/sql/functions.py:def from_csv(col, schema, options={}):
python/pyspark/sql/avro/functions.py:def from_avro(data, jsonFormatSchema, options={}):
```
```
fokkodriesprongFan spark % grep -R "=\[\]" python | grep def
python/pyspark/ml/tuning.py: def __init__(self, bestModel, avgMetrics=[], subModels=None):
python/pyspark/ml/tuning.py: def __init__(self, bestModel, validationMetrics=[], subModels=None):
```
### What changes were proposed in this pull request?
Removing the mutable default arguments.
### Why are the changes needed?
Removing the mutable default arguments, and changing the signature to `Optional[...]`.
### Does this PR introduce _any_ user-facing change?
No 👍
### How was this patch tested?
Using the Flake8 bugbear code analysis plugin.
Closes #29122 from Fokko/SPARK-32320.
Authored-by: Fokko Driesprong <fokko@apache.org>
Signed-off-by: Ruifeng Zheng <ruifengz@foxmail.com>
2020-12-07 20:35:36 -05:00
|
|
|
no_implicit_optional = True
|
|
|
|
|
2020-09-24 01:15:36 -04:00
|
|
|
[mypy-pyspark.cloudpickle.*]
|
|
|
|
ignore_errors = True
|
|
|
|
|
2020-11-24 19:27:04 -05:00
|
|
|
; Ignore missing imports for external untyped packages
|
|
|
|
|
2020-09-24 01:15:36 -04:00
|
|
|
[mypy-py4j.*]
|
|
|
|
ignore_missing_imports = True
|
|
|
|
|
|
|
|
[mypy-numpy]
|
|
|
|
ignore_missing_imports = True
|
|
|
|
|
|
|
|
[mypy-scipy.*]
|
|
|
|
ignore_missing_imports = True
|
|
|
|
|
|
|
|
[mypy-pandas.*]
|
|
|
|
ignore_missing_imports = True
|
|
|
|
|
2020-10-07 06:53:59 -04:00
|
|
|
[mypy-pyarrow.*]
|
|
|
|
ignore_missing_imports = True
|
|
|
|
|
|
|
|
[mypy-psutil.*]
|
2020-09-24 01:15:36 -04:00
|
|
|
ignore_missing_imports = True
|