[SPARK-36585][SQL][DOCS] Support setting "since" version in FunctionRegistry

### What changes were proposed in this pull request?

Spark 3.2.0 includes two new functions `regexp` and `regexp_like`, which are identical to `rlike`. However, in the generated documentation. the since versions of both functions are `1.0.0` since they are based on the expression `RLike`:

- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp
- https://dist.apache.org/repos/dist/dev/spark/v3.2.0-rc1-docs/_site/api/sql/index.html#regexp_like

This PR is to:
* Support setting `since` version in FunctionRegistry
* Correct the `since` version of `regexp` and `regexp_like`

### Why are the changes needed?

Correct the SQL doc
### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Run
```
sh sql/create-docs.sh
```
and check the SQL doc manually

Closes #33834 from gengliangwang/allowSQLFunVersion.

Authored-by: Gengliang Wang <gengliang@apache.org>
Signed-off-by: Gengliang Wang <gengliang@apache.org>
(cherry picked from commit 18143fb426)
Signed-off-by: Gengliang Wang <gengliang@apache.org>
This commit is contained in:
Gengliang Wang 2021-08-25 22:32:20 +08:00
parent fb38887e00
commit 464841224c

View file

@ -107,7 +107,9 @@ object FunctionRegistryBase {
* Return an expression info and a function builder for the function as defined by
* T using the given name.
*/
def build[T : ClassTag](name: String): (ExpressionInfo, Seq[Expression] => T) = {
def build[T : ClassTag](
name: String,
since: Option[String]): (ExpressionInfo, Seq[Expression] => T) = {
val runtimeClass = scala.reflect.classTag[T].runtimeClass
// For `RuntimeReplaceable`, skip the constructor with most arguments, which is the main
// constructor and contains non-parameter `child` and should not be used as function builder.
@ -150,13 +152,13 @@ object FunctionRegistryBase {
}
}
(expressionInfo(name), builder)
(expressionInfo(name, since), builder)
}
/**
* Creates an [[ExpressionInfo]] for the function as defined by T using the given name.
*/
def expressionInfo[T : ClassTag](name: String): ExpressionInfo = {
def expressionInfo[T : ClassTag](name: String, since: Option[String]): ExpressionInfo = {
val clazz = scala.reflect.classTag[T].runtimeClass
val df = clazz.getAnnotation(classOf[ExpressionDescription])
if (df != null) {
@ -170,7 +172,7 @@ object FunctionRegistryBase {
df.examples(),
df.note(),
df.group(),
df.since(),
since.getOrElse(df.since()),
df.deprecated(),
df.source())
} else {
@ -495,12 +497,12 @@ object FunctionRegistry {
expression[RegExpExtract]("regexp_extract"),
expression[RegExpExtractAll]("regexp_extract_all"),
expression[RegExpReplace]("regexp_replace"),
expression[RLike]("regexp_like", true),
expression[RLike]("regexp", true),
expression[StringRepeat]("repeat"),
expression[StringReplace]("replace"),
expression[Overlay]("overlay"),
expression[RLike]("rlike"),
expression[RLike]("regexp_like", true, Some("3.2.0")),
expression[RLike]("regexp", true, Some("3.2.0")),
expression[StringRPad]("rpad"),
expression[StringTrimRight]("rtrim"),
expression[Sentences]("sentences"),
@ -735,10 +737,19 @@ object FunctionRegistry {
val functionSet: Set[FunctionIdentifier] = builtin.listFunction().toSet
/** See usage above. */
private def expression[T <: Expression : ClassTag](name: String, setAlias: Boolean = false)
: (String, (ExpressionInfo, FunctionBuilder)) = {
val (expressionInfo, builder) = FunctionRegistryBase.build[T](name)
/**
* Create a SQL function builder and corresponding `ExpressionInfo`.
* @param name The function name.
* @param setAlias The alias name used in SQL representation string.
* @param since The Spark version since the function is added.
* @tparam T The actual expression class.
* @return (function name, (expression information, function builder))
*/
private def expression[T <: Expression : ClassTag](
name: String,
setAlias: Boolean = false,
since: Option[String] = None): (String, (ExpressionInfo, FunctionBuilder)) = {
val (expressionInfo, builder) = FunctionRegistryBase.build[T](name, since)
val newBuilder = (expressions: Seq[Expression]) => {
val expr = builder(expressions)
if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
@ -813,7 +824,7 @@ object TableFunctionRegistry {
private def logicalPlan[T <: LogicalPlan : ClassTag](name: String)
: (String, (ExpressionInfo, TableFunctionBuilder)) = {
val (info, builder) = FunctionRegistryBase.build[T](name)
val (info, builder) = FunctionRegistryBase.build[T](name, since = None)
val newBuilder = (expressions: Seq[Expression]) => {
try {
builder(expressions)