[SPARK-7358][SQL] Move DataFrame mathfunctions into functions
After a discussion on the user mailing list, it was decided to put all UDF's under `o.a.s.sql.functions` cc rxin Author: Burak Yavuz <brkyvz@gmail.com> Closes #5923 from brkyvz/move-math-funcs and squashes the following commits: a8dc3f7 [Burak Yavuz] address commentscf7a7bb[Burak Yavuz] [SPARK-7358] Move DataFrame mathfunctions into functions (cherry picked from commitba2b56614d) Signed-off-by: Reynold Xin <rxin@databricks.com>
This commit is contained in:
Burak Yavuz
Reynold Xin
parent
b5cd7dc1f0
commit
8aa6681d5f
|
|
@ -51,6 +51,19 @@ def _create_function(name, doc=""):
|
|||
return _
|
||||
|
||||
|
||||
def _create_binary_mathfunction(name, doc=""):
|
||||
""" Create a binary mathfunction by name"""
|
||||
def _(col1, col2):
|
||||
sc = SparkContext._active_spark_context
|
||||
# users might write ints for simplicity. This would throw an error on the JVM side.
|
||||
jc = getattr(sc._jvm.functions, name)(col1._jc if isinstance(col1, Column) else float(col1),
|
||||
col2._jc if isinstance(col2, Column) else float(col2))
|
||||
return Column(jc)
|
||||
_.__name__ = name
|
||||
_.__doc__ = doc
|
||||
return _
|
||||
|
||||
|
||||
_functions = {
|
||||
'lit': 'Creates a :class:`Column` of literal value.',
|
||||
'col': 'Returns a :class:`Column` based on the given column name.',
|
||||
|
|
@ -63,6 +76,34 @@ _functions = {
|
|||
'sqrt': 'Computes the square root of the specified float value.',
|
||||
'abs': 'Computes the absolute value.',
|
||||
|
||||
# unary math functions
|
||||
'acos': 'Computes the cosine inverse of the given value; the returned angle is in the range' +
|
||||
'0.0 through pi.',
|
||||
'asin': 'Computes the sine inverse of the given value; the returned angle is in the range' +
|
||||
'-pi/2 through pi/2.',
|
||||
'atan': 'Computes the tangent inverse of the given value.',
|
||||
'cbrt': 'Computes the cube-root of the given value.',
|
||||
'ceil': 'Computes the ceiling of the given value.',
|
||||
'cos': 'Computes the cosine of the given value.',
|
||||
'cosh': 'Computes the hyperbolic cosine of the given value.',
|
||||
'exp': 'Computes the exponential of the given value.',
|
||||
'expm1': 'Computes the exponential of the given value minus one.',
|
||||
'floor': 'Computes the floor of the given value.',
|
||||
'log': 'Computes the natural logarithm of the given value.',
|
||||
'log10': 'Computes the logarithm of the given value in Base 10.',
|
||||
'log1p': 'Computes the natural logarithm of the given value plus one.',
|
||||
'rint': 'Returns the double value that is closest in value to the argument and' +
|
||||
' is equal to a mathematical integer.',
|
||||
'signum': 'Computes the signum of the given value.',
|
||||
'sin': 'Computes the sine of the given value.',
|
||||
'sinh': 'Computes the hyperbolic sine of the given value.',
|
||||
'tan': 'Computes the tangent of the given value.',
|
||||
'tanh': 'Computes the hyperbolic tangent of the given value.',
|
||||
'toDegrees': 'Converts an angle measured in radians to an approximately equivalent angle ' +
|
||||
'measured in degrees.',
|
||||
'toRadians': 'Converts an angle measured in degrees to an approximately equivalent angle ' +
|
||||
'measured in radians.',
|
||||
|
||||
'max': 'Aggregate function: returns the maximum value of the expression in a group.',
|
||||
'min': 'Aggregate function: returns the minimum value of the expression in a group.',
|
||||
'first': 'Aggregate function: returns the first value in a group.',
|
||||
|
|
@ -74,10 +115,21 @@ _functions = {
|
|||
'sumDistinct': 'Aggregate function: returns the sum of distinct values in the expression.',
|
||||
}
|
||||
|
||||
# math functions that take two arguments as input
|
||||
_binary_mathfunctions = {
|
||||
'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x, y) to' +
|
||||
'polar coordinates (r, theta).',
|
||||
'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
|
||||
'pow': 'Returns the value of the first argument raised to the power of the second argument.'
|
||||
}
|
||||
|
||||
for _name, _doc in _functions.items():
|
||||
globals()[_name] = _create_function(_name, _doc)
|
||||
for _name, _doc in _binary_mathfunctions.items():
|
||||
globals()[_name] = _create_binary_mathfunction(_name, _doc)
|
||||
del _name, _doc
|
||||
__all__ += _functions.keys()
|
||||
__all__ += _binary_mathfunctions.keys()
|
||||
__all__.sort()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
A collection of builtin math functions
|
||||
"""
|
||||
|
||||
from pyspark import SparkContext
|
||||
from pyspark.sql.dataframe import Column
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
def _create_unary_mathfunction(name, doc=""):
|
||||
""" Create a unary mathfunction by name"""
|
||||
def _(col):
|
||||
sc = SparkContext._active_spark_context
|
||||
jc = getattr(sc._jvm.mathfunctions, name)(col._jc if isinstance(col, Column) else col)
|
||||
return Column(jc)
|
||||
_.__name__ = name
|
||||
_.__doc__ = doc
|
||||
return _
|
||||
|
||||
|
||||
def _create_binary_mathfunction(name, doc=""):
|
||||
""" Create a binary mathfunction by name"""
|
||||
def _(col1, col2):
|
||||
sc = SparkContext._active_spark_context
|
||||
# users might write ints for simplicity. This would throw an error on the JVM side.
|
||||
if type(col1) is int:
|
||||
col1 = col1 * 1.0
|
||||
if type(col2) is int:
|
||||
col2 = col2 * 1.0
|
||||
jc = getattr(sc._jvm.mathfunctions, name)(col1._jc if isinstance(col1, Column) else col1,
|
||||
col2._jc if isinstance(col2, Column) else col2)
|
||||
return Column(jc)
|
||||
_.__name__ = name
|
||||
_.__doc__ = doc
|
||||
return _
|
||||
|
||||
|
||||
# math functions are found under another object therefore, they need to be handled separately
|
||||
_mathfunctions = {
|
||||
'acos': 'Computes the cosine inverse of the given value; the returned angle is in the range' +
|
||||
'0.0 through pi.',
|
||||
'asin': 'Computes the sine inverse of the given value; the returned angle is in the range' +
|
||||
'-pi/2 through pi/2.',
|
||||
'atan': 'Computes the tangent inverse of the given value.',
|
||||
'cbrt': 'Computes the cube-root of the given value.',
|
||||
'ceil': 'Computes the ceiling of the given value.',
|
||||
'cos': 'Computes the cosine of the given value.',
|
||||
'cosh': 'Computes the hyperbolic cosine of the given value.',
|
||||
'exp': 'Computes the exponential of the given value.',
|
||||
'expm1': 'Computes the exponential of the given value minus one.',
|
||||
'floor': 'Computes the floor of the given value.',
|
||||
'log': 'Computes the natural logarithm of the given value.',
|
||||
'log10': 'Computes the logarithm of the given value in Base 10.',
|
||||
'log1p': 'Computes the natural logarithm of the given value plus one.',
|
||||
'rint': 'Returns the double value that is closest in value to the argument and' +
|
||||
' is equal to a mathematical integer.',
|
||||
'signum': 'Computes the signum of the given value.',
|
||||
'sin': 'Computes the sine of the given value.',
|
||||
'sinh': 'Computes the hyperbolic sine of the given value.',
|
||||
'tan': 'Computes the tangent of the given value.',
|
||||
'tanh': 'Computes the hyperbolic tangent of the given value.',
|
||||
'toDeg': 'Converts an angle measured in radians to an approximately equivalent angle ' +
|
||||
'measured in degrees.',
|
||||
'toRad': 'Converts an angle measured in degrees to an approximately equivalent angle ' +
|
||||
'measured in radians.'
|
||||
}
|
||||
|
||||
# math functions that take two arguments as input
|
||||
_binary_mathfunctions = {
|
||||
'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x, y) to' +
|
||||
'polar coordinates (r, theta).',
|
||||
'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
|
||||
'pow': 'Returns the value of the first argument raised to the power of the second argument.'
|
||||
}
|
||||
|
||||
for _name, _doc in _mathfunctions.items():
|
||||
globals()[_name] = _create_unary_mathfunction(_name, _doc)
|
||||
for _name, _doc in _binary_mathfunctions.items():
|
||||
globals()[_name] = _create_binary_mathfunction(_name, _doc)
|
||||
del _name, _doc
|
||||
__all__ += _mathfunctions.keys()
|
||||
__all__ += _binary_mathfunctions.keys()
|
||||
__all__.sort()
|
||||
|
|
@ -416,7 +416,7 @@ class SQLTests(ReusedPySparkTestCase):
|
|||
|
||||
def test_math_functions(self):
|
||||
df = self.sc.parallelize([Row(a=i, b=2 * i) for i in range(10)]).toDF()
|
||||
from pyspark.sql import mathfunctions as functions
|
||||
from pyspark.sql import functions
|
||||
import math
|
||||
|
||||
def get_values(l):
|
||||
|
|
|
|||
|
|
@ -1217,11 +1217,11 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
|
|||
unaryMathFunctionEvaluation(Tanh, math.tanh)
|
||||
}
|
||||
|
||||
test("toDeg") {
|
||||
test("toDegrees") {
|
||||
unaryMathFunctionEvaluation(ToDegrees, math.toDegrees)
|
||||
}
|
||||
|
||||
test("toRad") {
|
||||
test("toRadians") {
|
||||
unaryMathFunctionEvaluation(ToRadians, math.toRadians)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import org.apache.spark.annotation.Experimental
|
|||
import org.apache.spark.sql.catalyst.ScalaReflection
|
||||
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, Star}
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.mathfuncs._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.util.Utils
|
||||
|
||||
|
|
@ -35,6 +36,7 @@ import org.apache.spark.util.Utils
|
|||
* @groupname agg_funcs Aggregate functions
|
||||
* @groupname sort_funcs Sorting functions
|
||||
* @groupname normal_funcs Non-aggregate functions
|
||||
* @groupname math_funcs Math functions
|
||||
* @groupname Ungrouped Support functions for DataFrames.
|
||||
*/
|
||||
@Experimental
|
||||
|
|
@ -436,6 +438,489 @@ object functions {
|
|||
*/
|
||||
def upper(e: Column): Column = Upper(e.expr)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Math Functions
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/**
|
||||
* Computes the cosine inverse of the given value; the returned angle is in the range
|
||||
* 0.0 through pi.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def acos(e: Column): Column = Acos(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cosine inverse of the given column; the returned angle is in the range
|
||||
* 0.0 through pi.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def acos(columnName: String): Column = acos(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the sine inverse of the given value; the returned angle is in the range
|
||||
* -pi/2 through pi/2.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def asin(e: Column): Column = Asin(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the sine inverse of the given column; the returned angle is in the range
|
||||
* -pi/2 through pi/2.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def asin(columnName: String): Column = asin(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the tangent inverse of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan(e: Column): Column = Atan(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the tangent inverse of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan(columnName: String): Column = atan(Column(columnName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(leftName: String, rightName: String): Column =
|
||||
atan2(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes the cube-root of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cbrt(e: Column): Column = Cbrt(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cube-root of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cbrt(columnName: String): Column = cbrt(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the ceiling of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def ceil(e: Column): Column = Ceil(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the ceiling of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def ceil(columnName: String): Column = ceil(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the cosine of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cos(e: Column): Column = Cos(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cosine of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cos(columnName: String): Column = cos(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic cosine of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cosh(e: Column): Column = Cosh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic cosine of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def cosh(columnName: String): Column = cosh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def exp(e: Column): Column = Exp(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def exp(columnName: String): Column = exp(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given value minus one.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def expm1(e: Column): Column = Expm1(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def expm1(columnName: String): Column = expm1(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the floor of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def floor(e: Column): Column = Floor(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the floor of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def floor(columnName: String): Column = floor(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(leftName: String, rightName: String): Column =
|
||||
hypot(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log(e: Column): Column = Log(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log(columnName: String): Column = log(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the logarithm of the given value in Base 10.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log10(e: Column): Column = Log10(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the logarithm of the given value in Base 10.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log10(columnName: String): Column = log10(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given value plus one.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log1p(e: Column): Column = Log1p(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given column plus one.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def log1p(columnName: String): Column = log1p(Column(columnName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the double value that is closest in value to the argument and
|
||||
* is equal to a mathematical integer.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def rint(e: Column): Column = Rint(e.expr)
|
||||
|
||||
/**
|
||||
* Returns the double value that is closest in value to the argument and
|
||||
* is equal to a mathematical integer.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def rint(columnName: String): Column = rint(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the signum of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def signum(e: Column): Column = Signum(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the signum of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def signum(columnName: String): Column = signum(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the sine of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def sin(e: Column): Column = Sin(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the sine of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def sin(columnName: String): Column = sin(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic sine of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def sinh(e: Column): Column = Sinh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic sine of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def sinh(columnName: String): Column = sinh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the tangent of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def tan(e: Column): Column = Tan(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the tangent of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def tan(columnName: String): Column = tan(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic tangent of the given value.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def tanh(e: Column): Column = Tanh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic tangent of the given column.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def tanh(columnName: String): Column = tanh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def toDegrees(e: Column): Column = ToDegrees(e.expr)
|
||||
|
||||
/**
|
||||
* Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def toDegrees(columnName: String): Column = toDegrees(Column(columnName))
|
||||
|
||||
/**
|
||||
* Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def toRadians(e: Column): Column = ToRadians(e.expr)
|
||||
|
||||
/**
|
||||
* Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
||||
*
|
||||
* @group math_funcs
|
||||
*/
|
||||
def toRadians(columnName: String): Column = toRadians(Column(columnName))
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
|
|
|||
|
|
@ -1,383 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.spark.sql
|
||||
|
||||
import scala.language.implicitConversions
|
||||
|
||||
import org.apache.spark.annotation.Experimental
|
||||
import org.apache.spark.sql.catalyst.expressions._
|
||||
import org.apache.spark.sql.catalyst.expressions.mathfuncs._
|
||||
import org.apache.spark.sql.functions.lit
|
||||
|
||||
/**
|
||||
* :: Experimental ::
|
||||
* Mathematical Functions available for [[DataFrame]].
|
||||
*/
|
||||
@Experimental
|
||||
// scalastyle:off
|
||||
object mathfunctions {
|
||||
// scalastyle:on
|
||||
|
||||
private[this] implicit def toColumn(expr: Expression): Column = Column(expr)
|
||||
|
||||
/**
|
||||
* Computes the cosine inverse of the given value; the returned angle is in the range
|
||||
* 0.0 through pi.
|
||||
*/
|
||||
def acos(e: Column): Column = Acos(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cosine inverse of the given column; the returned angle is in the range
|
||||
* 0.0 through pi.
|
||||
*/
|
||||
def acos(columnName: String): Column = acos(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the sine inverse of the given value; the returned angle is in the range
|
||||
* -pi/2 through pi/2.
|
||||
*/
|
||||
def asin(e: Column): Column = Asin(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the sine inverse of the given column; the returned angle is in the range
|
||||
* -pi/2 through pi/2.
|
||||
*/
|
||||
def asin(columnName: String): Column = asin(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the tangent inverse of the given value.
|
||||
*/
|
||||
def atan(e: Column): Column = Atan(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the tangent inverse of the given column.
|
||||
*/
|
||||
def atan(columnName: String): Column = atan(Column(columnName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(leftName: String, rightName: String): Column =
|
||||
atan2(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).=
|
||||
*/
|
||||
def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Returns the angle theta from the conversion of rectangular coordinates (x, y) to
|
||||
* polar coordinates (r, theta).
|
||||
*/
|
||||
def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes the cube-root of the given value.
|
||||
*/
|
||||
def cbrt(e: Column): Column = Cbrt(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cube-root of the given column.
|
||||
*/
|
||||
def cbrt(columnName: String): Column = cbrt(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the ceiling of the given value.
|
||||
*/
|
||||
def ceil(e: Column): Column = Ceil(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the ceiling of the given column.
|
||||
*/
|
||||
def ceil(columnName: String): Column = ceil(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the cosine of the given value.
|
||||
*/
|
||||
def cos(e: Column): Column = Cos(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the cosine of the given column.
|
||||
*/
|
||||
def cos(columnName: String): Column = cos(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic cosine of the given value.
|
||||
*/
|
||||
def cosh(e: Column): Column = Cosh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic cosine of the given column.
|
||||
*/
|
||||
def cosh(columnName: String): Column = cosh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given value.
|
||||
*/
|
||||
def exp(e: Column): Column = Exp(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given column.
|
||||
*/
|
||||
def exp(columnName: String): Column = exp(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given value minus one.
|
||||
*/
|
||||
def expm1(e: Column): Column = Expm1(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the exponential of the given column.
|
||||
*/
|
||||
def expm1(columnName: String): Column = expm1(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the floor of the given value.
|
||||
*/
|
||||
def floor(e: Column): Column = Floor(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the floor of the given column.
|
||||
*/
|
||||
def floor(columnName: String): Column = floor(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(leftName: String, rightName: String): Column =
|
||||
hypot(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
|
||||
*/
|
||||
def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given value.
|
||||
*/
|
||||
def log(e: Column): Column = Log(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given column.
|
||||
*/
|
||||
def log(columnName: String): Column = log(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the logarithm of the given value in Base 10.
|
||||
*/
|
||||
def log10(e: Column): Column = Log10(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the logarithm of the given value in Base 10.
|
||||
*/
|
||||
def log10(columnName: String): Column = log10(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given value plus one.
|
||||
*/
|
||||
def log1p(e: Column): Column = Log1p(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the natural logarithm of the given column plus one.
|
||||
*/
|
||||
def log1p(columnName: String): Column = log1p(Column(columnName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
|
||||
|
||||
/**
|
||||
* Returns the value of the first argument raised to the power of the second argument.
|
||||
*/
|
||||
def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
|
||||
|
||||
/**
|
||||
* Returns the double value that is closest in value to the argument and
|
||||
* is equal to a mathematical integer.
|
||||
*/
|
||||
def rint(e: Column): Column = Rint(e.expr)
|
||||
|
||||
/**
|
||||
* Returns the double value that is closest in value to the argument and
|
||||
* is equal to a mathematical integer.
|
||||
*/
|
||||
def rint(columnName: String): Column = rint(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the signum of the given value.
|
||||
*/
|
||||
def signum(e: Column): Column = Signum(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the signum of the given column.
|
||||
*/
|
||||
def signum(columnName: String): Column = signum(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the sine of the given value.
|
||||
*/
|
||||
def sin(e: Column): Column = Sin(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the sine of the given column.
|
||||
*/
|
||||
def sin(columnName: String): Column = sin(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic sine of the given value.
|
||||
*/
|
||||
def sinh(e: Column): Column = Sinh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic sine of the given column.
|
||||
*/
|
||||
def sinh(columnName: String): Column = sinh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the tangent of the given value.
|
||||
*/
|
||||
def tan(e: Column): Column = Tan(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the tangent of the given column.
|
||||
*/
|
||||
def tan(columnName: String): Column = tan(Column(columnName))
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic tangent of the given value.
|
||||
*/
|
||||
def tanh(e: Column): Column = Tanh(e.expr)
|
||||
|
||||
/**
|
||||
* Computes the hyperbolic tangent of the given column.
|
||||
*/
|
||||
def tanh(columnName: String): Column = tanh(Column(columnName))
|
||||
|
||||
/**
|
||||
* Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
||||
*/
|
||||
def toDeg(e: Column): Column = ToDegrees(e.expr)
|
||||
|
||||
/**
|
||||
* Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
||||
*/
|
||||
def toDeg(columnName: String): Column = toDeg(Column(columnName))
|
||||
|
||||
/**
|
||||
* Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
||||
*/
|
||||
def toRad(e: Column): Column = ToRadians(e.expr)
|
||||
|
||||
/**
|
||||
* Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
||||
*/
|
||||
def toRad(columnName: String): Column = toRad(Column(columnName))
|
||||
}
|
||||
|
|
@ -39,7 +39,6 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
import static org.apache.spark.sql.functions.*;
|
||||
import static org.apache.spark.sql.mathfunctions.*;
|
||||
|
||||
public class JavaDataFrameSuite {
|
||||
private transient JavaSparkContext jsc;
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ package org.apache.spark.sql
|
|||
|
||||
import java.lang.{Double => JavaDouble}
|
||||
|
||||
import org.apache.spark.sql.functions.lit
|
||||
import org.apache.spark.sql.mathfunctions._
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.test.TestSQLContext
|
||||
import org.apache.spark.sql.test.TestSQLContext.implicits._
|
||||
|
||||
|
|
@ -159,11 +158,11 @@ class MathExpressionsSuite extends QueryTest {
|
|||
}
|
||||
|
||||
test("toDeg") {
|
||||
testOneToOneMathFunction(toDeg, math.toDegrees)
|
||||
testOneToOneMathFunction(toDegrees, math.toDegrees)
|
||||
}
|
||||
|
||||
test("toRad") {
|
||||
testOneToOneMathFunction(toRad, math.toRadians)
|
||||
testOneToOneMathFunction(toRadians, math.toRadians)
|
||||
}
|
||||
|
||||
test("cbrt") {
|
||||
|
|
|
|||
Loading…
Reference in a new issue