[SPARK-19975][PYTHON][SQL] Add map_keys and map_values functions to Python
## What changes were proposed in this pull request? This fix tries to address the issue in SPARK-19975 where we have `map_keys` and `map_values` functions in SQL yet there is no Python equivalent functions. This fix adds `map_keys` and `map_values` functions to Python. ## How was this patch tested? This fix is tested manually (See Python docs for examples). Author: Yong Tang <yong.tang.github@outlook.com> Closes #17328 from yongtang/SPARK-19975.
This commit is contained in:
parent
66a792cd88
commit
e5387018e7
|
@ -1855,6 +1855,46 @@ def sort_array(col, asc=True):
|
|||
return Column(sc._jvm.functions.sort_array(_to_java_column(col), asc))
|
||||
|
||||
|
||||
@since(2.3)
|
||||
def map_keys(col):
|
||||
"""
|
||||
Collection function: Returns an unordered array containing the keys of the map.
|
||||
|
||||
:param col: name of column or expression
|
||||
|
||||
>>> from pyspark.sql.functions import map_keys
|
||||
>>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
|
||||
>>> df.select(map_keys("data").alias("keys")).show()
|
||||
+------+
|
||||
| keys|
|
||||
+------+
|
||||
|[1, 2]|
|
||||
+------+
|
||||
"""
|
||||
sc = SparkContext._active_spark_context
|
||||
return Column(sc._jvm.functions.map_keys(_to_java_column(col)))
|
||||
|
||||
|
||||
@since(2.3)
|
||||
def map_values(col):
|
||||
"""
|
||||
Collection function: Returns an unordered array containing the values of the map.
|
||||
|
||||
:param col: name of column or expression
|
||||
|
||||
>>> from pyspark.sql.functions import map_values
|
||||
>>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data")
|
||||
>>> df.select(map_values("data").alias("values")).show()
|
||||
+------+
|
||||
|values|
|
||||
+------+
|
||||
|[a, b]|
|
||||
+------+
|
||||
"""
|
||||
sc = SparkContext._active_spark_context
|
||||
return Column(sc._jvm.functions.map_values(_to_java_column(col)))
|
||||
|
||||
|
||||
# ---------------------------- User Defined Function ----------------------------------
|
||||
|
||||
def _wrap_function(sc, func, returnType):
|
||||
|
|
|
@ -3161,6 +3161,20 @@ object functions {
|
|||
*/
|
||||
def sort_array(e: Column, asc: Boolean): Column = withExpr { SortArray(e.expr, lit(asc).expr) }
|
||||
|
||||
/**
|
||||
* Returns an unordered array containing the keys of the map.
|
||||
* @group collection_funcs
|
||||
* @since 2.3.0
|
||||
*/
|
||||
def map_keys(e: Column): Column = withExpr { MapKeys(e.expr) }
|
||||
|
||||
/**
|
||||
* Returns an unordered array containing the values of the map.
|
||||
* @group collection_funcs
|
||||
* @since 2.3.0
|
||||
*/
|
||||
def map_values(e: Column): Column = withExpr { MapValues(e.expr) }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
|
Loading…
Reference in a new issue