ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-5578][SQL][DataFrame] Provide a convenient way for Scala users to use UDFs

Browse source 
A more convenient way to define user-defined functions.

Author: Reynold Xin <rxin@databricks.com>

Closes #4345 from rxin/defineUDF and squashes the following commits:

639c0f8 [Reynold Xin] udf tests.
0a0b339 [Reynold Xin] defineUDF -> udf.
b452b8d [Reynold Xin] Fix UDF registration.
d2e42c3 [Reynold Xin] SQLContext.udf.register() returns a UserDefinedFunction also.
4333605 [Reynold Xin] [SQL][DataFrame] defineUDF.
This commit is contained in:
Reynold Xin 2015-02-03 20:07:46 -08:00
parent e380d2d46c
commit 1077f2e1de
11 changed files with 261 additions and 430 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
View file

@ -132,14 +132,14 @@ class LogisticRegressionModel private[ml] (
override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = { override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = {
transformSchema(dataset.schema, paramMap, logging = true) transformSchema(dataset.schema, paramMap, logging = true)
val map = this.paramMap ++ paramMap val map = this.paramMap ++ paramMap
val scoreFunction: Vector => Double = (v) => { val scoreFunction = udf((v: Vector) => {
val margin = BLAS.dot(v, weights) val margin = BLAS.dot(v, weights)
1.0 / (1.0 + math.exp(-margin)) 1.0 / (1.0 + math.exp(-margin))
} } : Double)
val t = map(threshold) val t = map(threshold)
val predictFunction: Double => Double = (score) => { if (score > t) 1.0 else 0.0 } val predictFunction = udf((score: Double) => { if (score > t) 1.0 else 0.0 } : Double)
dataset dataset
.select($"*", callUDF(scoreFunction, col(map(featuresCol))).as(map(scoreCol))) .select($"*", scoreFunction(col(map(featuresCol))).as(map(scoreCol)))
.select($"*", callUDF(predictFunction, col(map(scoreCol))).as(map(predictionCol))) .select($"*", predictFunction(col(map(scoreCol))).as(map(predictionCol)))
} }
} }

6
mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
View file

@ -81,10 +81,8 @@ class StandardScalerModel private[ml] (
override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = { override def transform(dataset: DataFrame, paramMap: ParamMap): DataFrame = {
transformSchema(dataset.schema, paramMap, logging = true) transformSchema(dataset.schema, paramMap, logging = true)
val map = this.paramMap ++ paramMap val map = this.paramMap ++ paramMap
val scale: (Vector) => Vector = (v) => { val scale = udf((v: Vector) => { scaler.transform(v) } : Vector)
scaler.transform(v) dataset.select($"*", scale(col(map(inputCol))).as(map(outputCol)))
}
dataset.select($"*", callUDF(scale, col(map(inputCol))).as(map(outputCol)))
} }
private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = { private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {

14
mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
View file

@ -126,22 +126,20 @@ class ALSModel private[ml] (
val map = this.paramMap ++ paramMap val map = this.paramMap ++ paramMap
val users = userFactors.toDataFrame("id", "features") val users = userFactors.toDataFrame("id", "features")
val items = itemFactors.toDataFrame("id", "features") val items = itemFactors.toDataFrame("id", "features")
val predict: (Seq[Float], Seq[Float]) => Float = (userFeatures, itemFeatures) => {
// Register a UDF for DataFrame, and then
// create a new column named map(predictionCol) by running the predict UDF.
val predict = udf((userFeatures: Seq[Float], itemFeatures: Seq[Float]) => {
if (userFeatures != null && itemFeatures != null) { if (userFeatures != null && itemFeatures != null) {
blas.sdot(k, userFeatures.toArray, 1, itemFeatures.toArray, 1) blas.sdot(k, userFeatures.toArray, 1, itemFeatures.toArray, 1)
} else { } else {
Float.NaN Float.NaN
} }
} } : Float)
val inputColumns = dataset.schema.fieldNames
val prediction = callUDF(predict, users("features"), items("features")).as(map(predictionCol))
val outputColumns = inputColumns.map(f => dataset(f)) :+ prediction
dataset dataset
.join(users, dataset(map(userCol)) === users("id"), "left") .join(users, dataset(map(userCol)) === users("id"), "left")
.join(items, dataset(map(itemCol)) === items("id"), "left") .join(items, dataset(map(itemCol)) === items("id"), "left")
.select(outputColumns: _*) .select(dataset("*"), predict(users("features"), items("features")).as(map(predictionCol)))
// TODO: Just use a dataset("*")
// .select(dataset("*"), prediction)
} }
override private[ml] def transformSchema(schema: StructType, paramMap: ParamMap): StructType = { override private[ml] def transformSchema(schema: StructType, paramMap: ParamMap): StructType = {

4
sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
View file

@ -54,10 +54,10 @@ private[sql] object DataFrame {
* }}} * }}}
* *
* Note that the [[Column]] type can also be manipulated through its various functions. * Note that the [[Column]] type can also be manipulated through its various functions.
* {{ * {{{
* // The following creates a new column that increases everybody's age by 10. * // The following creates a new column that increases everybody's age by 10.
* people("age") + 10 // in Scala * people("age") + 10 // in Scala
* }} * }}}
* *
* A more concrete example: * A more concrete example:
* {{{ * {{{

2
sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
View file

@ -329,7 +329,7 @@ private[sql] class DataFrameImpl protected[sql](
override def save(path: String): Unit = { override def save(path: String): Unit = {
val dataSourceName = sqlContext.conf.defaultDataSourceName val dataSourceName = sqlContext.conf.defaultDataSourceName
save(dataSourceName, ("path" -> path)) save(dataSourceName, "path" -> path)
} }
override def save( override def save(

194
sql/core/src/main/scala/org/apache/spark/sql/Dsl.scala
View file

@ -186,15 +186,13 @@ object Dsl {
(0 to 22).map { x => (0 to 22).map { x =>
val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
val args = (1 to x).map(i => s"arg$i: Column").mkString(", ")
val argsInUdf = (1 to x).map(i => s"arg$i.expr").mkString(", ")
println(s""" println(s"""
/** /**
* Call a Scala function of ${x} arguments as user-defined function (UDF), and automatically * Defines a user-defined function of ${x} arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[$typeTags](f: Function$x[$types]${if (args.length > 0) ", " + args else ""}): Column = { def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq($argsInUdf)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
}""") }""")
} }
@ -214,187 +212,187 @@ object Dsl {
} }
*/ */
/** /**
* Call a Scala function of 0 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 0 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag](f: Function0[RT]): Column = { def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq()) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 1 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 1 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT], arg1: Column): Column = { def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 2 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 2 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT], arg1: Column, arg2: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 3 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 3 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT], arg1: Column, arg2: Column, arg3: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 4 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 4 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 5 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 5 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 6 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 6 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 7 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 7 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 8 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 8 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 9 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 9 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 10 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 10 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 11 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 11 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 12 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 12 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 13 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 13 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](f: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](f: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 14 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 14 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](f: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](f: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 15 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 15 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](f: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](f: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 16 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 16 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](f: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](f: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 17 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 17 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](f: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](f: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 18 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 18 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](f: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](f: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 19 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 19 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](f: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](f: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 20 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 20 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](f: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](f: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 21 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 21 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](f: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column, arg21: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](f: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr, arg21.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
/** /**
* Call a Scala function of 22 arguments as user-defined function (UDF), and automatically * Defines a user-defined function of 22 arguments as user-defined function (UDF).
* infer the data types based on the function's signature. * The data types are automatically inferred based on the function's signature.
*/ */
def callUDF[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](f: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT], arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column, arg11: Column, arg12: Column, arg13: Column, arg14: Column, arg15: Column, arg16: Column, arg17: Column, arg18: Column, arg19: Column, arg20: Column, arg21: Column, arg22: Column): Column = { def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](f: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
ScalaUdf(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr, arg11.expr, arg12.expr, arg13.expr, arg14.expr, arg15.expr, arg16.expr, arg17.expr, arg18.expr, arg19.expr, arg20.expr, arg21.expr, arg22.expr)) UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType)
} }
////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////

405
sql/core/src/main/scala/org/apache/spark/sql/UdfRegistration.scala → sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
View file

@ -78,20 +78,21 @@ class UDFRegistration(sqlContext: SQLContext) extends Logging {
// scalastyle:off // scalastyle:off
/* registerFunction 0-22 were generated by this script /* register 0-22 were generated by this script
(0 to 22).map { x => (0 to 22).map { x =>
val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"}) val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
val typeTags = (1 to x).map(i => s"A${i}: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _) val typeTags = (1 to x).map(i => s"A${i}: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
val argDocs = (1 to x).map(i => s" * @tparam A$i type of the UDF argument at position $i.").foldLeft("")(_ + "\n" + _)
println(s""" println(s"""
/** /**
* Register a Scala closure of ${x} arguments as user-defined function (UDF). * Register a Scala closure of ${x} arguments as user-defined function (UDF).
* @tparam RT return type of UDF.$argDocs * @tparam RT return type of UDF.
*/ */
def register[$typeTags](name: String, func: Function$x[$types]): Unit = { def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
}""") }""")
} }
@ -116,462 +117,258 @@ class UDFRegistration(sqlContext: SQLContext) extends Logging {
* Register a Scala closure of 0 arguments as user-defined function (UDF). * Register a Scala closure of 0 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
*/ */
def register[RT: TypeTag](name: String, func: Function0[RT]): Unit = { def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 1 arguments as user-defined function (UDF). * Register a Scala closure of 1 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
*/ */
def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 2 arguments as user-defined function (UDF). * Register a Scala closure of 2 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 3 arguments as user-defined function (UDF). * Register a Scala closure of 3 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 4 arguments as user-defined function (UDF). * Register a Scala closure of 4 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 5 arguments as user-defined function (UDF). * Register a Scala closure of 5 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 6 arguments as user-defined function (UDF). * Register a Scala closure of 6 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 7 arguments as user-defined function (UDF). * Register a Scala closure of 7 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 8 arguments as user-defined function (UDF). * Register a Scala closure of 8 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 9 arguments as user-defined function (UDF). * Register a Scala closure of 9 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 10 arguments as user-defined function (UDF). * Register a Scala closure of 10 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 11 arguments as user-defined function (UDF). * Register a Scala closure of 11 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 12 arguments as user-defined function (UDF). * Register a Scala closure of 12 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 13 arguments as user-defined function (UDF). * Register a Scala closure of 13 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 14 arguments as user-defined function (UDF). * Register a Scala closure of 14 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 15 arguments as user-defined function (UDF). * Register a Scala closure of 15 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 16 arguments as user-defined function (UDF). * Register a Scala closure of 16 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 17 arguments as user-defined function (UDF). * Register a Scala closure of 17 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 18 arguments as user-defined function (UDF). * Register a Scala closure of 18 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
* @tparam A18 type of the UDF argument at position 18.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 19 arguments as user-defined function (UDF). * Register a Scala closure of 19 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
* @tparam A18 type of the UDF argument at position 18.
* @tparam A19 type of the UDF argument at position 19.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 20 arguments as user-defined function (UDF). * Register a Scala closure of 20 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
* @tparam A18 type of the UDF argument at position 18.
* @tparam A19 type of the UDF argument at position 19.
* @tparam A20 type of the UDF argument at position 20.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 21 arguments as user-defined function (UDF). * Register a Scala closure of 21 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
* @tparam A18 type of the UDF argument at position 18.
* @tparam A19 type of the UDF argument at position 19.
* @tparam A20 type of the UDF argument at position 20.
* @tparam A21 type of the UDF argument at position 21.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
/** /**
* Register a Scala closure of 22 arguments as user-defined function (UDF). * Register a Scala closure of 22 arguments as user-defined function (UDF).
* @tparam RT return type of UDF. * @tparam RT return type of UDF.
* @tparam A1 type of the UDF argument at position 1.
* @tparam A2 type of the UDF argument at position 2.
* @tparam A3 type of the UDF argument at position 3.
* @tparam A4 type of the UDF argument at position 4.
* @tparam A5 type of the UDF argument at position 5.
* @tparam A6 type of the UDF argument at position 6.
* @tparam A7 type of the UDF argument at position 7.
* @tparam A8 type of the UDF argument at position 8.
* @tparam A9 type of the UDF argument at position 9.
* @tparam A10 type of the UDF argument at position 10.
* @tparam A11 type of the UDF argument at position 11.
* @tparam A12 type of the UDF argument at position 12.
* @tparam A13 type of the UDF argument at position 13.
* @tparam A14 type of the UDF argument at position 14.
* @tparam A15 type of the UDF argument at position 15.
* @tparam A16 type of the UDF argument at position 16.
* @tparam A17 type of the UDF argument at position 17.
* @tparam A18 type of the UDF argument at position 18.
* @tparam A19 type of the UDF argument at position 19.
* @tparam A20 type of the UDF argument at position 20.
* @tparam A21 type of the UDF argument at position 21.
* @tparam A22 type of the UDF argument at position 22.
*/ */
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): Unit = { def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
def builder(e: Seq[Expression]) = ScalaUdf(func, ScalaReflection.schemaFor[RT].dataType, e) val dataType = ScalaReflection.schemaFor[RT].dataType
def builder(e: Seq[Expression]) = ScalaUdf(func, dataType, e)
functionRegistry.registerFunction(name, builder) functionRegistry.registerFunction(name, builder)
UserDefinedFunction(func, dataType)
} }
//////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////
/** /**
* Register a user-defined function with 1 arguments. * Register a user-defined function with 1 arguments.
*/ */

39
sql/core/src/main/scala/org/apache/spark/sql/UserDefinedFunction.scala Normal file
View file

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.expressions.ScalaUdf
import org.apache.spark.sql.types.DataType
/**
* A user-defined function. To create one, use the `udf` functions in [[Dsl]].
* As an example:
* {{{
* // Defined a UDF that returns true or false based on some numeric score.
* val predict = udf((score: Double) => if (score > 0.5) true else false)
*
* // Projects a column that adds a prediction column based on the score column.
* df.select( predict(df("score")) )
* }}}
*/
case class UserDefinedFunction(f: AnyRef, dataType: DataType) {
def apply(exprs: Column*): Column = {
Column(ScalaUdf(f, dataType, exprs.map(_.expr)))
}
}

6
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
View file

@ -21,7 +21,7 @@ import org.apache.spark.sql.Dsl._
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
/* Implicits */ /* Implicits */
import org.apache.spark.sql.test.TestSQLContext._ import org.apache.spark.sql.test.TestSQLContext.{createDataFrame, logicalPlanToSparkQuery}
import scala.language.postfixOps import scala.language.postfixOps
@ -280,11 +280,11 @@ class DataFrameSuite extends QueryTest {
} }
test("udf") { test("udf") {
val foo = (a: Int, b: String) => a.toString + b val foo = udf((a: Int, b: String) => a.toString + b)
checkAnswer( checkAnswer(
// SELECT *, foo(key, value) FROM testData // SELECT *, foo(key, value) FROM testData
testData.select($"*", callUDF(foo, 'key, 'value)).limit(3), testData.select($"*", foo('key, 'value)).limit(3),
Row(1, "1", "11") :: Row(2, "2", "22") :: Row(3, "3", "33") :: Nil Row(1, "1", "11") :: Row(2, "2", "22") :: Row(3, "3", "33") :: Nil
) )
} }

6
sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
View file

@ -19,6 +19,7 @@ package org.apache.spark.sql
import java.util.TimeZone import java.util.TimeZone
import org.apache.spark.sql.test.TestSQLContext
import org.scalatest.BeforeAndAfterAll import org.scalatest.BeforeAndAfterAll
import org.apache.spark.sql.Dsl._ import org.apache.spark.sql.Dsl._
@ -26,9 +27,8 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
/* Implicits */
import org.apache.spark.sql.TestData._ import org.apache.spark.sql.TestData._
import org.apache.spark.sql.test.TestSQLContext._ import org.apache.spark.sql.test.TestSQLContext.{udf => _, _}
class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
@ -794,7 +794,7 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
} }
test("SPARK-3371 Renaming a function expression with group by gives error") { test("SPARK-3371 Renaming a function expression with group by gives error") {
udf.register("len", (s: String) => s.length) TestSQLContext.udf.register("len", (s: String) => s.length)
checkAnswer( checkAnswer(
sql("SELECT len(value) as temp FROM testData WHERE key = 1 group by len(value)"), sql("SELECT len(value) as temp FROM testData WHERE key = 1 group by len(value)"),
Row(1)) Row(1))

5
sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
View file

@ -21,7 +21,8 @@ import scala.beans.{BeanInfo, BeanProperty}
import org.apache.spark.rdd.RDD import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Dsl._ import org.apache.spark.sql.Dsl._
import org.apache.spark.sql.test.TestSQLContext._ import org.apache.spark.sql.test.TestSQLContext
import org.apache.spark.sql.test.TestSQLContext.{udf => _, _}
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
@ -83,7 +84,7 @@ class UserDefinedTypeSuite extends QueryTest {
} }
test("UDTs and UDFs") { test("UDTs and UDFs") {
udf.register("testType", (d: MyDenseVector) => d.isInstanceOf[MyDenseVector]) TestSQLContext.udf.register("testType", (d: MyDenseVector) => d.isInstanceOf[MyDenseVector])
pointsRDD.registerTempTable("points") pointsRDD.registerTempTable("points")
checkAnswer( checkAnswer(
sql("SELECT testType(features) from points"), sql("SELECT testType(features) from points"),