[SPARK-15620][SQL] Fix transformed dataset attributes revolve failure
## What changes were proposed in this pull request? Join on transformed dataset has attributes conflicts, which make query execution failure, for example: ``` val dataset = Seq(1, 2, 3).toDs val mappedDs = dataset.map(_ + 1) mappedDs.as("t1").joinWith(mappedDs.as("t2"), $"t1.value" === $"t2.value").show() ``` will throw exception: ``` org.apache.spark.sql.AnalysisException: cannot resolve '`t1.value`' given input columns: [value]; at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:62) at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:59) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:287) at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:287) ``` ## How was this patch tested? Unit test. Author: jerryshao <sshao@hortonworks.com> Closes #13399 from jerryshao/SPARK-15620.
This commit is contained in:
parent
6dddb70c38
commit
8288e16a5a
|
@ -524,6 +524,10 @@ class Analyzer(
|
|||
val newVersion = oldVersion.newInstance()
|
||||
(oldVersion, newVersion)
|
||||
|
||||
case oldVersion: SerializeFromObject
|
||||
if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
|
||||
(oldVersion, oldVersion.copy(serializer = oldVersion.serializer.map(_.newInstance())))
|
||||
|
||||
// Handle projects that create conflicting aliases.
|
||||
case oldVersion @ Project(projectList, _)
|
||||
if findAliases(projectList).intersect(conflictingAttributes).nonEmpty =>
|
||||
|
|
|
@ -769,6 +769,16 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
|
|||
}
|
||||
}
|
||||
|
||||
test("mapped dataset should resolve duplicated attributes for self join") {
|
||||
val ds = Seq(1, 2, 3).toDS().map(_ + 1)
|
||||
val ds1 = ds.as("d1")
|
||||
val ds2 = ds.as("d2")
|
||||
|
||||
checkDataset(ds1.joinWith(ds2, $"d1.value" === $"d2.value"), (2, 2), (3, 3), (4, 4))
|
||||
checkDataset(ds1.intersect(ds2), 2, 3, 4)
|
||||
checkDataset(ds1.except(ds1))
|
||||
}
|
||||
|
||||
test("SPARK-15441: Dataset outer join") {
|
||||
val left = Seq(ClassData("a", 1), ClassData("b", 2)).toDS().as("left")
|
||||
val right = Seq(ClassData("x", 2), ClassData("y", 3)).toDS().as("right")
|
||||
|
|
Loading…
Reference in a new issue