ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-35871][SQL] Literal.create(value, dataType) should support fields

Browse source 
### What changes were proposed in this pull request?
Current Literal.create(data, dataType) for Period to YearMonthIntervalType and Duration to DayTimeIntervalType is not correct.

if data type is Period/Duration, it will create converter of default YearMonthIntervalType/DayTimeIntervalType,  then the result is not correct, this pr fix this bug.

### Why are the changes needed?
Fix  bug when use Literal.create()

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT

Closes #33056 from AngersZhuuuu/SPARK-35871.

Authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
Angerszhuuuu 2021-06-24 17:36:48 +03:00 committed by Max Gekk
parent d40a1a2552
commit de35675c61
2 changed files with 31 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
View file

@ -153,7 +153,13 @@ object Literal {
def fromObject(obj: Any): Literal = new Literal(obj, ObjectType(obj.getClass)) def fromObject(obj: Any): Literal = new Literal(obj, ObjectType(obj.getClass))
def create(v: Any, dataType: DataType): Literal = { def create(v: Any, dataType: DataType): Literal = {
Literal(CatalystTypeConverters.convertToCatalyst(v), dataType) dataType match {
case _: YearMonthIntervalType if v.isInstanceOf[Period] =>
Literal(CatalystTypeConverters.createToCatalystConverter(dataType)(v), dataType)
case _: DayTimeIntervalType if v.isInstanceOf[Duration] =>
Literal(CatalystTypeConverters.createToCatalystConverter(dataType)(v), dataType)
case _ => Literal(CatalystTypeConverters.convertToCatalyst(v), dataType)
}
} }
def create[T : TypeTag](v: T): Literal = Try { def create[T : TypeTag](v: T): Literal = Try {

24
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
View file

@ -32,6 +32,8 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._ import org.apache.spark.sql.types._
import org.apache.spark.sql.types.DayTimeIntervalType._
import org.apache.spark.sql.types.YearMonthIntervalType._
import org.apache.spark.unsafe.types.CalendarInterval import org.apache.spark.unsafe.types.CalendarInterval
class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
@ -432,4 +434,26 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
assert(literal.toString === expected) assert(literal.toString === expected)
} }
} }
test("SPARK-35871: Literal.create(value, dataType) should support fields") {
val period = Period.ofMonths(13)
DataTypeTestUtils.yearMonthIntervalTypes.foreach { dt =>
val result = dt.endField match {
case YEAR => 12
case MONTH => 13
}
checkEvaluation(Literal.create(period, dt), result)
}
val duration = Duration.ofSeconds(86400 + 3600 + 60 + 1)
DataTypeTestUtils.dayTimeIntervalTypes.foreach { dt =>
val result = dt.endField match {
case DAY => 86400000000L
case HOUR => 90000000000L
case MINUTE => 90060000000L
case SECOND => 90061000000L
}
checkEvaluation(Literal.create(duration, dt), result)
}
}
} }