### What changes were proposed in this pull request? This PR fix an issue that virtual operators (`||`, `!=`, `<>`, `between` and `case`) are absent from the Spark SQL Built-in functions document. ### Why are the changes needed? The document should explain about all the supported built-in operators. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Built the document with `SKIP_SCALADOC=1 SKIP_RDOC=1 SKIP_PYTHONDOC=1 bundler exec jekyll build` and then, confirmed the document. ![neq1](https://user-images.githubusercontent.com/4736016/111192859-e2e76380-85fc-11eb-89c9-75916a5e856a.png) ![neq2](https://user-images.githubusercontent.com/4736016/111192874-e7ac1780-85fc-11eb-9a9b-c504265b373f.png) ![between](https://user-images.githubusercontent.com/4736016/111192898-eda1f880-85fc-11eb-992d-cf80c544ec27.png) ![case](https://user-images.githubusercontent.com/4736016/111192918-f266ac80-85fc-11eb-9306-5dbc413a0cdb.png) ![double_pipe](https://user-images.githubusercontent.com/4736016/111192952-fb577e00-85fc-11eb-932e-385e5c2a5205.png) Closes #31841 from sarutak/builtin-op-doc. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
329 lines
10 KiB
Python
329 lines
10 KiB
Python
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
import os
|
|
from collections import namedtuple
|
|
|
|
from pyspark.java_gateway import launch_gateway
|
|
|
|
|
|
ExpressionInfo = namedtuple(
|
|
"ExpressionInfo", "className name usage arguments examples note since deprecated")
|
|
|
|
_virtual_operator_infos = [
|
|
ExpressionInfo(
|
|
className="",
|
|
name="!=",
|
|
usage="expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, " +
|
|
"or false otherwise.",
|
|
arguments="\n Arguments:\n " +
|
|
"""* expr1, expr2 - the two expressions must be same type or can be casted to
|
|
a common type, and must be a type that can be used in equality comparison.
|
|
Map type is not supported. For complex types such array/struct,
|
|
the data types of fields must be orderable.""",
|
|
examples="\n Examples:\n " +
|
|
"> SELECT 1 != 2;\n " +
|
|
" true\n " +
|
|
"> SELECT 1 != '2';\n " +
|
|
" true\n " +
|
|
"> SELECT true != NULL;\n " +
|
|
" NULL\n " +
|
|
"> SELECT NULL != NULL;\n " +
|
|
" NULL",
|
|
note="",
|
|
since="1.0.0",
|
|
deprecated=""),
|
|
ExpressionInfo(
|
|
className="",
|
|
name="<>",
|
|
usage="expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`, " +
|
|
"or false otherwise.",
|
|
arguments="\n Arguments:\n " +
|
|
"""* expr1, expr2 - the two expressions must be same type or can be casted to
|
|
a common type, and must be a type that can be used in equality comparison.
|
|
Map type is not supported. For complex types such array/struct,
|
|
the data types of fields must be orderable.""",
|
|
examples="\n Examples:\n " +
|
|
"> SELECT 1 != 2;\n " +
|
|
" true\n " +
|
|
"> SELECT 1 != '2';\n " +
|
|
" true\n " +
|
|
"> SELECT true != NULL;\n " +
|
|
" NULL\n " +
|
|
"> SELECT NULL != NULL;\n " +
|
|
" NULL",
|
|
note="",
|
|
since="1.0.0",
|
|
deprecated=""),
|
|
ExpressionInfo(
|
|
className="",
|
|
name="between",
|
|
usage="expr1 [NOT] BETWEEN expr2 AND expr3 - " +
|
|
"evaluate if `expr1` is [not] in between `expr2` and `expr3`.",
|
|
arguments="",
|
|
examples="\n Examples:\n " +
|
|
"> SELECT col1 FROM VALUES 1, 3, 5, 7 WHERE col1 BETWEEN 2 AND 5;\n " +
|
|
" 3\n " +
|
|
" 5",
|
|
note="",
|
|
since="1.0.0",
|
|
deprecated=""),
|
|
ExpressionInfo(
|
|
className="",
|
|
name="case",
|
|
usage="CASE expr1 WHEN expr2 THEN expr3 " +
|
|
"[WHEN expr4 THEN expr5]* [ELSE expr6] END - " +
|
|
"When `expr1` = `expr2`, returns `expr3`; " +
|
|
"when `expr1` = `expr4`, return `expr5`; else return `expr6`.",
|
|
arguments="\n Arguments:\n " +
|
|
"* expr1 - the expression which is one operand of comparison.\n " +
|
|
"* expr2, expr4 - the expressions each of which is the other " +
|
|
" operand of comparison.\n " +
|
|
"* expr3, expr5, expr6 - the branch value expressions and else value expression" +
|
|
" should all be same type or coercible to a common type.",
|
|
examples="\n Examples:\n " +
|
|
"> SELECT CASE col1 WHEN 1 THEN 'one' " +
|
|
"WHEN 2 THEN 'two' ELSE '?' END FROM VALUES 1, 2, 3;\n " +
|
|
" one\n " +
|
|
" two\n " +
|
|
" ?\n " +
|
|
"> SELECT CASE col1 WHEN 1 THEN 'one' " +
|
|
"WHEN 2 THEN 'two' END FROM VALUES 1, 2, 3;\n " +
|
|
" one\n " +
|
|
" two\n " +
|
|
" NULL",
|
|
note="",
|
|
since="1.0.1",
|
|
deprecated=""),
|
|
ExpressionInfo(
|
|
className="",
|
|
name="||",
|
|
usage="expr1 || expr2 - Returns the concatenation of `expr1` and `expr2`.",
|
|
arguments="",
|
|
examples="\n Examples:\n " +
|
|
"> SELECT 'Spark' || 'SQL';\n " +
|
|
" SparkSQL\n " +
|
|
"> SELECT array(1, 2, 3) || array(4, 5) || array(6);\n " +
|
|
" [1,2,3,4,5,6]",
|
|
note="\n || for arrays is available since 2.4.0.\n",
|
|
since="2.3.0",
|
|
deprecated="")
|
|
]
|
|
|
|
|
|
def _list_function_infos(jvm):
|
|
"""
|
|
Returns a list of function information via JVM. Sorts wrapped expression infos by name
|
|
and returns them.
|
|
"""
|
|
|
|
jinfos = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listBuiltinFunctionInfos()
|
|
infos = _virtual_operator_infos
|
|
for jinfo in jinfos:
|
|
name = jinfo.getName()
|
|
usage = jinfo.getUsage()
|
|
usage = usage.replace("_FUNC_", name) if usage is not None else usage
|
|
infos.append(ExpressionInfo(
|
|
className=jinfo.getClassName(),
|
|
name=name,
|
|
usage=usage,
|
|
arguments=jinfo.getArguments().replace("_FUNC_", name),
|
|
examples=jinfo.getExamples().replace("_FUNC_", name),
|
|
note=jinfo.getNote().replace("_FUNC_", name),
|
|
since=jinfo.getSince(),
|
|
deprecated=jinfo.getDeprecated()))
|
|
return sorted(infos, key=lambda i: i.name)
|
|
|
|
|
|
def _make_pretty_usage(usage):
|
|
"""
|
|
Makes the usage description pretty and returns a formatted string if `usage`
|
|
is not an empty string. Otherwise, returns None.
|
|
"""
|
|
|
|
if usage is not None and usage.strip() != "":
|
|
usage = "\n".join(map(lambda u: u.strip(), usage.split("\n")))
|
|
return "%s\n\n" % usage
|
|
|
|
|
|
def _make_pretty_arguments(arguments):
|
|
"""
|
|
Makes the arguments description pretty and returns a formatted string if `arguments`
|
|
starts with the argument prefix. Otherwise, returns None.
|
|
|
|
Expected input:
|
|
|
|
Arguments:
|
|
* arg0 - ...
|
|
...
|
|
* arg0 - ...
|
|
...
|
|
|
|
Expected output:
|
|
**Arguments:**
|
|
|
|
* arg0 - ...
|
|
...
|
|
* arg0 - ...
|
|
...
|
|
|
|
"""
|
|
|
|
if arguments.startswith("\n Arguments:"):
|
|
arguments = "\n".join(map(lambda u: u[6:], arguments.strip().split("\n")[1:]))
|
|
return "**Arguments:**\n\n%s\n\n" % arguments
|
|
|
|
|
|
def _make_pretty_examples(examples):
|
|
"""
|
|
Makes the examples description pretty and returns a formatted string if `examples`
|
|
starts with the example prefix. Otherwise, returns None.
|
|
|
|
Expected input:
|
|
|
|
Examples:
|
|
> SELECT ...;
|
|
...
|
|
> SELECT ...;
|
|
...
|
|
|
|
Expected output:
|
|
**Examples:**
|
|
|
|
```
|
|
> SELECT ...;
|
|
...
|
|
> SELECT ...;
|
|
...
|
|
```
|
|
|
|
"""
|
|
|
|
if examples.startswith("\n Examples:"):
|
|
examples = "\n".join(map(lambda u: u[6:], examples.strip().split("\n")[1:]))
|
|
return "**Examples:**\n\n```\n%s\n```\n\n" % examples
|
|
|
|
|
|
def _make_pretty_note(note):
|
|
"""
|
|
Makes the note description pretty and returns a formatted string if `note` is not
|
|
an empty string. Otherwise, returns None.
|
|
|
|
Expected input:
|
|
|
|
...
|
|
|
|
Expected output:
|
|
**Note:**
|
|
|
|
...
|
|
|
|
"""
|
|
|
|
if note != "":
|
|
note = "\n".join(map(lambda n: n[4:], note.split("\n")))
|
|
return "**Note:**\n%s\n" % note
|
|
|
|
|
|
def _make_pretty_deprecated(deprecated):
|
|
"""
|
|
Makes the deprecated description pretty and returns a formatted string if `deprecated`
|
|
is not an empty string. Otherwise, returns None.
|
|
|
|
Expected input:
|
|
|
|
...
|
|
|
|
Expected output:
|
|
**Deprecated:**
|
|
|
|
...
|
|
|
|
"""
|
|
|
|
if deprecated != "":
|
|
deprecated = "\n".join(map(lambda n: n[4:], deprecated.split("\n")))
|
|
return "**Deprecated:**\n%s\n" % deprecated
|
|
|
|
|
|
def generate_sql_api_markdown(jvm, path):
|
|
"""
|
|
Generates a markdown file after listing the function information. The output file
|
|
is created in `path`.
|
|
|
|
Expected output:
|
|
### NAME
|
|
|
|
USAGE
|
|
|
|
**Arguments:**
|
|
|
|
ARGUMENTS
|
|
|
|
**Examples:**
|
|
|
|
```
|
|
EXAMPLES
|
|
```
|
|
|
|
**Note:**
|
|
|
|
NOTE
|
|
|
|
**Since:** SINCE
|
|
|
|
**Deprecated:**
|
|
|
|
DEPRECATED
|
|
|
|
<br/>
|
|
|
|
"""
|
|
|
|
with open(path, 'w') as mdfile:
|
|
mdfile.write("# Built-in Functions\n\n")
|
|
for info in _list_function_infos(jvm):
|
|
name = info.name
|
|
usage = _make_pretty_usage(info.usage)
|
|
arguments = _make_pretty_arguments(info.arguments)
|
|
examples = _make_pretty_examples(info.examples)
|
|
note = _make_pretty_note(info.note)
|
|
since = info.since
|
|
deprecated = _make_pretty_deprecated(info.deprecated)
|
|
|
|
mdfile.write("### %s\n\n" % name)
|
|
if usage is not None:
|
|
mdfile.write("%s\n\n" % usage.strip())
|
|
if arguments is not None:
|
|
mdfile.write(arguments)
|
|
if examples is not None:
|
|
mdfile.write(examples)
|
|
if note is not None:
|
|
mdfile.write(note)
|
|
if since is not None and since != "":
|
|
mdfile.write("**Since:** %s\n\n" % since.strip())
|
|
if deprecated is not None:
|
|
mdfile.write(deprecated)
|
|
mdfile.write("<br/>\n\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
jvm = launch_gateway().jvm
|
|
spark_root_dir = os.path.dirname(os.path.dirname(__file__))
|
|
markdown_file_path = os.path.join(spark_root_dir, "sql/docs/index.md")
|
|
generate_sql_api_markdown(jvm, markdown_file_path)
|