[SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry
### What changes were proposed in this pull request?
Spark 3.2.0 includes two new functions `regexp` and `regexp_like`, which are identical to `rlike`. However, in the generated documentation. the since versions of both functions are `1.0.0` since they are based on the expression `RLike`:
- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp
- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp_like
This PR is to:
* Support setting `since` version in FunctionRegistry
* Correct the `since` version of `regexp` and `regexp_like`
### Why are the changes needed?
Correct the SQL doc
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Run
```
sh sql/create-docs.sh
```
and check the SQL doc manually
Closes #33834 from gengliangwang/allowSQLFunVersion.
Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
(cherry picked from commit 18143fb426
)
Signed-off-by: Gengliang Wang <gengliang@apache.org>
This commit is contained in:
parent
fb38887e00
commit
464841224c
|
@ -107,7 +107,9 @@ object FunctionRegistryBase {
|
||||||
* Return an expression info and a function builder for the function as defined by
|
* Return an expression info and a function builder for the function as defined by
|
||||||
* T using the given name.
|
* T using the given name.
|
||||||
*/
|
*/
|
||||||
def build[T : ClassTag](name: String): (ExpressionInfo, Seq[Expression] => T) = {
|
def build[T : ClassTag](
|
||||||
|
name: String,
|
||||||
|
since: Option[String]): (ExpressionInfo, Seq[Expression] => T) = {
|
||||||
val runtimeClass = scala.reflect.classTag[T].runtimeClass
|
val runtimeClass = scala.reflect.classTag[T].runtimeClass
|
||||||
// For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
|
// For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
|
||||||
// constructor and contains non-parameter `child` and should not be used as function builder.
|
// constructor and contains non-parameter `child` and should not be used as function builder.
|
||||||
|
@ -150,13 +152,13 @@ object FunctionRegistryBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
(expressionInfo(name), builder)
|
(expressionInfo(name, since), builder)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an [[ExpressionInfo]] for the function as defined by T using the given name.
|
* Creates an [[ExpressionInfo]] for the function as defined by T using the given name.
|
||||||
*/
|
*/
|
||||||
def expressionInfo[T : ClassTag](name: String): ExpressionInfo = {
|
def expressionInfo[T : ClassTag](name: String, since: Option[String]): ExpressionInfo = {
|
||||||
val clazz = scala.reflect.classTag[T].runtimeClass
|
val clazz = scala.reflect.classTag[T].runtimeClass
|
||||||
val df = clazz.getAnnotation(classOf[ExpressionDescription])
|
val df = clazz.getAnnotation(classOf[ExpressionDescription])
|
||||||
if (df != null) {
|
if (df != null) {
|
||||||
|
@ -170,7 +172,7 @@ object FunctionRegistryBase {
|
||||||
df.examples(),
|
df.examples(),
|
||||||
df.note(),
|
df.note(),
|
||||||
df.group(),
|
df.group(),
|
||||||
df.since(),
|
since.getOrElse(df.since()),
|
||||||
df.deprecated(),
|
df.deprecated(),
|
||||||
df.source())
|
df.source())
|
||||||
} else {
|
} else {
|
||||||
|
@ -495,12 +497,12 @@ object FunctionRegistry {
|
||||||
expression[RegExpExtract]("regexp_extract"),
|
expression[RegExpExtract]("regexp_extract"),
|
||||||
expression[RegExpExtractAll]("regexp_extract_all"),
|
expression[RegExpExtractAll]("regexp_extract_all"),
|
||||||
expression[RegExpReplace]("regexp_replace"),
|
expression[RegExpReplace]("regexp_replace"),
|
||||||
expression[RLike]("regexp_like", true),
|
|
||||||
expression[RLike]("regexp", true),
|
|
||||||
expression[StringRepeat]("repeat"),
|
expression[StringRepeat]("repeat"),
|
||||||
expression[StringReplace]("replace"),
|
expression[StringReplace]("replace"),
|
||||||
expression[Overlay]("overlay"),
|
expression[Overlay]("overlay"),
|
||||||
expression[RLike]("rlike"),
|
expression[RLike]("rlike"),
|
||||||
|
expression[RLike]("regexp_like", true, Some("3.2.0")),
|
||||||
|
expression[RLike]("regexp", true, Some("3.2.0")),
|
||||||
expression[StringRPad]("rpad"),
|
expression[StringRPad]("rpad"),
|
||||||
expression[StringTrimRight]("rtrim"),
|
expression[StringTrimRight]("rtrim"),
|
||||||
expression[Sentences]("sentences"),
|
expression[Sentences]("sentences"),
|
||||||
|
@ -735,10 +737,19 @@ object FunctionRegistry {
|
||||||
|
|
||||||
val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
|
val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
|
||||||
|
|
||||||
/** See usage above. */
|
/**
|
||||||
private def expression[T <: Expression : ClassTag](name: String, setAlias: Boolean = false)
|
* Create a SQL function builder and corresponding `ExpressionInfo`.
|
||||||
: (String, (ExpressionInfo, FunctionBuilder)) = {
|
* @param name The function name.
|
||||||
val (expressionInfo, builder) = FunctionRegistryBase.build[T](name)
|
* @param setAlias The alias name used in SQL representation string.
|
||||||
|
* @param since The Spark version since the function is added.
|
||||||
|
* @tparam T The actual expression class.
|
||||||
|
* @return (function name, (expression information, function builder))
|
||||||
|
*/
|
||||||
|
private def expression[T <: Expression : ClassTag](
|
||||||
|
name: String,
|
||||||
|
setAlias: Boolean = false,
|
||||||
|
since: Option[String] = None): (String, (ExpressionInfo, FunctionBuilder)) = {
|
||||||
|
val (expressionInfo, builder) = FunctionRegistryBase.build[T](name, since)
|
||||||
val newBuilder = (expressions: Seq[Expression]) => {
|
val newBuilder = (expressions: Seq[Expression]) => {
|
||||||
val expr = builder(expressions)
|
val expr = builder(expressions)
|
||||||
if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
|
if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
|
||||||
|
@ -813,7 +824,7 @@ object TableFunctionRegistry {
|
||||||
|
|
||||||
private def logicalPlan[T <: LogicalPlan : ClassTag](name: String)
|
private def logicalPlan[T <: LogicalPlan : ClassTag](name: String)
|
||||||
: (String, (ExpressionInfo, TableFunctionBuilder)) = {
|
: (String, (ExpressionInfo, TableFunctionBuilder)) = {
|
||||||
val (info, builder) = FunctionRegistryBase.build[T](name)
|
val (info, builder) = FunctionRegistryBase.build[T](name, since = None)
|
||||||
val newBuilder = (expressions: Seq[Expression]) => {
|
val newBuilder = (expressions: Seq[Expression]) => {
|
||||||
try {
|
try {
|
||||||
builder(expressions)
|
builder(expressions)
|
||||||
|
|
Loading…
Reference in a new issue