From bdabf60fb4a61b0eef95144f2c54477a10ea849f Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Mon, 2 Nov 2020 10:10:24 -0800 Subject: [PATCH] [SPARK-33299][SQL][DOCS] Don't mention schemas in JSON format in docs for `from_json` ### What changes were proposed in this pull request? Remove the JSON formatted schema from comments for `from_json()` in Scala/Python APIs. Closes #30201 ### Why are the changes needed? Schemas in JSON format is internal (not documented). It shouldn't be recommenced for usage. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? By linters. Closes #30226 from MaxGekk/from_json-common-schema-parsing-2. Authored-by: Max Gekk Signed-off-by: Dongjoon Hyun --- python/pyspark/sql/functions.py | 3 +-- .../src/main/scala/org/apache/spark/sql/functions.scala | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 69fdf220f1..c349ae5cf4 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -2847,8 +2847,7 @@ def from_json(col, schema, options={}): :param schema: a StructType or ArrayType of StructType to use when parsing the json column. :param options: options to control parsing. accepts the same options as the json datasource - .. note:: Since Spark 2.3, the DDL-formatted string or a JSON format string is also - supported for ``schema``. + .. note:: Since Spark 2.3, the DDL-formatted string is also supported for ``schema``. >>> from pyspark.sql.types import * >>> data = [(1, '''{"a": 1}''')] diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index ffa97c20c3..6bb9f7871e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -4077,9 +4077,7 @@ object functions { * Returns `null`, in the case of an unparseable string. * * @param e a string column containing JSON data. - * @param schema the schema to use when parsing the json string as a json string. In Spark 2.1, - * the user-provided schema has to be in JSON format. Since Spark 2.2, the DDL - * format is also supported for the schema. + * @param schema the schema as a DDL-formatted string. * * @group collection_funcs * @since 2.1.0 @@ -4094,8 +4092,7 @@ object functions { * Returns `null`, in the case of an unparseable string. * * @param e a string column containing JSON data. - * @param schema the schema to use when parsing the json string as a json string, it could be a - * JSON format string or a DDL-formatted string. + * @param schema the schema as a DDL-formatted string. * * @group collection_funcs * @since 2.3.0