2020-09-24 01:15:36 -04:00
|
|
|
#
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
|
|
|
|
|
|
|
from pyspark.resource.requests import ( # noqa: F401
|
|
|
|
ExecutorResourceRequest as ExecutorResourceRequest,
|
|
|
|
ExecutorResourceRequests as ExecutorResourceRequests,
|
|
|
|
TaskResourceRequest as TaskResourceRequest,
|
|
|
|
TaskResourceRequests as TaskResourceRequests,
|
|
|
|
)
|
[SPARK-32320][PYSPARK] Remove mutable default arguments
This is bad practice, and might lead to unexpected behaviour:
https://florimond.dev/blog/articles/2018/08/python-mutable-defaults-are-the-source-of-all-evil/
```
fokkodriesprongFan spark % grep -R "={}" python | grep def
python/pyspark/resource/profile.py: def __init__(self, _java_resource_profile=None, _exec_req={}, _task_req={}):
python/pyspark/sql/functions.py:def from_json(col, schema, options={}):
python/pyspark/sql/functions.py:def to_json(col, options={}):
python/pyspark/sql/functions.py:def schema_of_json(json, options={}):
python/pyspark/sql/functions.py:def schema_of_csv(csv, options={}):
python/pyspark/sql/functions.py:def to_csv(col, options={}):
python/pyspark/sql/functions.py:def from_csv(col, schema, options={}):
python/pyspark/sql/avro/functions.py:def from_avro(data, jsonFormatSchema, options={}):
```
```
fokkodriesprongFan spark % grep -R "=\[\]" python | grep def
python/pyspark/ml/tuning.py: def __init__(self, bestModel, avgMetrics=[], subModels=None):
python/pyspark/ml/tuning.py: def __init__(self, bestModel, validationMetrics=[], subModels=None):
```
### What changes were proposed in this pull request?
Removing the mutable default arguments.
### Why are the changes needed?
Removing the mutable default arguments, and changing the signature to `Optional[...]`.
### Does this PR introduce _any_ user-facing change?
No 👍
### How was this patch tested?
Using the Flake8 bugbear code analysis plugin.
Closes #29122 from Fokko/SPARK-32320.
Authored-by: Fokko Driesprong <fokko@apache.org>
Signed-off-by: Ruifeng Zheng <ruifengz@foxmail.com>
2020-12-07 20:35:36 -05:00
|
|
|
from typing import overload, Dict, Union, Optional
|
2020-10-07 21:32:30 -04:00
|
|
|
from py4j.java_gateway import JavaObject # type: ignore[import]
|
2020-09-24 01:15:36 -04:00
|
|
|
|
|
|
|
class ResourceProfile:
|
2020-10-07 21:32:30 -04:00
|
|
|
@overload
|
2020-09-24 01:15:36 -04:00
|
|
|
def __init__(
|
|
|
|
self,
|
2020-10-07 21:32:30 -04:00
|
|
|
_java_resource_profile: JavaObject,
|
|
|
|
) -> None: ...
|
|
|
|
@overload
|
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
_java_resource_profile: None = ...,
|
[SPARK-32320][PYSPARK] Remove mutable default arguments
This is bad practice, and might lead to unexpected behaviour:
https://florimond.dev/blog/articles/2018/08/python-mutable-defaults-are-the-source-of-all-evil/
```
fokkodriesprongFan spark % grep -R "={}" python | grep def
python/pyspark/resource/profile.py: def __init__(self, _java_resource_profile=None, _exec_req={}, _task_req={}):
python/pyspark/sql/functions.py:def from_json(col, schema, options={}):
python/pyspark/sql/functions.py:def to_json(col, options={}):
python/pyspark/sql/functions.py:def schema_of_json(json, options={}):
python/pyspark/sql/functions.py:def schema_of_csv(csv, options={}):
python/pyspark/sql/functions.py:def to_csv(col, options={}):
python/pyspark/sql/functions.py:def from_csv(col, schema, options={}):
python/pyspark/sql/avro/functions.py:def from_avro(data, jsonFormatSchema, options={}):
```
```
fokkodriesprongFan spark % grep -R "=\[\]" python | grep def
python/pyspark/ml/tuning.py: def __init__(self, bestModel, avgMetrics=[], subModels=None):
python/pyspark/ml/tuning.py: def __init__(self, bestModel, validationMetrics=[], subModels=None):
```
### What changes were proposed in this pull request?
Removing the mutable default arguments.
### Why are the changes needed?
Removing the mutable default arguments, and changing the signature to `Optional[...]`.
### Does this PR introduce _any_ user-facing change?
No 👍
### How was this patch tested?
Using the Flake8 bugbear code analysis plugin.
Closes #29122 from Fokko/SPARK-32320.
Authored-by: Fokko Driesprong <fokko@apache.org>
Signed-off-by: Ruifeng Zheng <ruifengz@foxmail.com>
2020-12-07 20:35:36 -05:00
|
|
|
_exec_req: Optional[Dict[str, ExecutorResourceRequest]] = ...,
|
|
|
|
_task_req: Optional[Dict[str, TaskResourceRequest]] = ...,
|
2020-09-24 01:15:36 -04:00
|
|
|
) -> None: ...
|
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def id(self) -> int: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def taskResources(self) -> Dict[str, TaskResourceRequest]: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def executorResources(self) -> Dict[str, ExecutorResourceRequest]: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
|
|
|
|
class ResourceProfileBuilder:
|
|
|
|
def __init__(self) -> None: ...
|
2020-10-07 21:32:30 -04:00
|
|
|
def require(
|
|
|
|
self, resourceRequest: Union[ExecutorResourceRequest, TaskResourceRequests]
|
2020-11-24 19:27:04 -05:00
|
|
|
) -> ResourceProfileBuilder: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
def clearExecutorResourceRequests(self) -> None: ...
|
|
|
|
def clearTaskResourceRequests(self) -> None: ...
|
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def taskResources(self) -> Dict[str, TaskResourceRequest]: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def executorResources(self) -> Dict[str, ExecutorResourceRequest]: ...
|
2020-09-24 01:15:36 -04:00
|
|
|
@property
|
2020-10-07 21:32:30 -04:00
|
|
|
def build(self) -> ResourceProfile: ...
|