[SPARK-36208][SQL][3.2] SparkScriptTransformation should support ANSI interval types
### What changes were proposed in this pull request?
This PR changes `BaseScriptTransformationExec` for `SparkScriptTransformationExec` to support ANSI interval types.
### Why are the changes needed?
`SparkScriptTransformationExec` support `CalendarIntervalType` so it's better to support ANSI interval types as well.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New test.
Authored-by: Kousuke Saruta <sarutakoss.nttdata.com>
Signed-off-by: Max Gekk <max.gekkgmail.com>
(cherry picked from commit f56c7b71ff
)
Signed-off-by: Max Gekk <max.gekkgmail.com>
Closes #33463 from MaxGekk/sarutak_script-transformation-interval-3.2.
Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
This commit is contained in:
parent
99eb3ff226
commit
468165ae52
|
@ -223,6 +223,14 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
|
|||
case CalendarIntervalType => wrapperConvertException(
|
||||
data => IntervalUtils.stringToInterval(UTF8String.fromString(data)),
|
||||
converter)
|
||||
case YearMonthIntervalType(start, end) => wrapperConvertException(
|
||||
data => IntervalUtils.monthsToPeriod(
|
||||
IntervalUtils.castStringToYMInterval(UTF8String.fromString(data), start, end)),
|
||||
converter)
|
||||
case DayTimeIntervalType(start, end) => wrapperConvertException(
|
||||
data => IntervalUtils.microsToDuration(
|
||||
IntervalUtils.castStringToDTInterval(UTF8String.fromString(data), start, end)),
|
||||
converter)
|
||||
case _: ArrayType | _: MapType | _: StructType =>
|
||||
val complexTypeFactory = JsonToStructs(attr.dataType,
|
||||
ioschema.outputSerdeProps.toMap, Literal(null), Some(conf.sessionLocalTimeZone))
|
||||
|
|
|
@ -633,6 +633,27 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
test("SPARK-36208: TRANSFORM should support ANSI interval (no serde)") {
|
||||
assume(TestUtils.testCommandAvailable("python"))
|
||||
withTempView("v") {
|
||||
val df = Seq(
|
||||
(Period.of(1, 2, 0), Duration.ofDays(1).plusHours(2).plusMinutes(3).plusSeconds(4))
|
||||
).toDF("ym", "dt")
|
||||
|
||||
checkAnswer(
|
||||
df,
|
||||
(child: SparkPlan) => createScriptTransformationExec(
|
||||
script = "cat",
|
||||
output = Seq(
|
||||
AttributeReference("ym", YearMonthIntervalType())(),
|
||||
AttributeReference("dt", DayTimeIntervalType())()),
|
||||
child = child,
|
||||
ioschema = defaultIOSchema
|
||||
),
|
||||
df.select($"ym", $"dt").collect())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {
|
||||
|
|
Loading…
Reference in a new issue