diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index a5b2d8be7a..e6c312abf1 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -92,6 +92,8 @@ license: | - In Spark 3.2, `CREATE TABLE AS SELECT` with non-empty `LOCATION` will throw `AnalysisException`. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.allowNonEmptyLocationInCTAS` to `true`. - In Spark 3.2, special datetime values such as `epoch`, `today`, `yesterday`, `tomorrow`, and `now` are supported in typed literals only, for instance, `select timestamp'now'`. In Spark 3.1 and 3.0, such special values are supported in any casts of strings to dates/timestamps. To keep these special values as dates/timestamps in Spark 3.1 and 3.0, you should replace them manually, e.g. `if (c in ('now', 'today'), current_date(), cast(c as date))`. + + - In Spark 3.2, `FloatType` is mapped to `FLOAT` in MySQL. Prior to this, it used to be mapped to `REAL`, which is by default a synonym to `DOUBLE PRECISION` in MySQL. ## Upgrading from Spark SQL 3.0 to 3.1 diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala index 71bba6f110..81b816449f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala @@ -20,7 +20,8 @@ package org.apache.spark.sql.jdbc import java.sql.{SQLFeatureNotSupportedException, Types} import java.util.Locale -import org.apache.spark.sql.types.{BooleanType, DataType, LongType, MetadataBuilder} +import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils +import org.apache.spark.sql.types.{BooleanType, DataType, FloatType, LongType, MetadataBuilder} private case object MySQLDialect extends JdbcDialect { @@ -94,4 +95,11 @@ private case object MySQLDialect extends JdbcDialect { override def getTableCommentQuery(table: String, comment: String): String = { s"ALTER TABLE $table COMMENT = '$comment'" } + + override def getJDBCType(dt: DataType): Option[JdbcType] = dt match { + // See SPARK-35446: MySQL treats REAL as a synonym to DOUBLE by default + // We override getJDBCType so that FloatType is mapped to FLOAT instead + case FloatType => Option(JdbcType("FLOAT", java.sql.Types.FLOAT)) + case _ => JdbcUtils.getCommonJDBCType(dt) + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 5865ff245a..f85c479d60 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -899,6 +899,11 @@ class JDBCSuite extends QueryTest Option(TimestampType)) } + test("SPARK-35446: MySQLDialect type mapping of float") { + val mySqlDialect = JdbcDialects.get("jdbc:mysql://127.0.0.1/db") + assert(mySqlDialect.getJDBCType(FloatType).map(_.databaseTypeDefinition).get == "FLOAT") + } + test("PostgresDialect type mapping") { val Postgres = JdbcDialects.get("jdbc:postgresql://127.0.0.1/db") val md = new MetadataBuilder().putLong("scale", 0)