[SPARK-24131][PYSPARK] Add majorMinorVersion API to PySpark for determining Spark versions
## What changes were proposed in this pull request? We need to determine Spark major and minor versions in PySpark. We can add a `majorMinorVersion` API to PySpark which is similar to the Scala API in `VersionUtils.majorMinorVersion`. ## How was this patch tested? Added tests. Author: Liang-Chi Hsieh <viirya@gmail.com> Closes #21203 from viirya/SPARK-24131.
This commit is contained in:
parent
6782359a04
commit
e15850be6e
|
@ -16,6 +16,7 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
import re
|
||||
import sys
|
||||
import inspect
|
||||
from py4j.protocol import Py4JJavaError
|
||||
|
@ -61,6 +62,26 @@ def _get_argspec(f):
|
|||
return argspec
|
||||
|
||||
|
||||
def majorMinorVersion(version):
|
||||
"""
|
||||
Get major and minor version numbers for given Spark version string.
|
||||
|
||||
>>> version = "2.4.0"
|
||||
>>> majorMinorVersion(version)
|
||||
(2, 4)
|
||||
|
||||
>>> version = "abc"
|
||||
>>> majorMinorVersion(version) is None
|
||||
True
|
||||
|
||||
"""
|
||||
m = re.search('^(\d+)\.(\d+)(\..*)?$', version)
|
||||
if m is None:
|
||||
return None
|
||||
else:
|
||||
return (int(m.group(1)), int(m.group(2)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import doctest
|
||||
(failure_count, test_count) = doctest.testmod()
|
||||
|
|
Loading…
Reference in a new issue