ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Revert "[SPARK-27278][SQL] Optimize GetMapValue when the map is a foldable and the key is not"

Browse source 
This reverts commit 5888b15d9c.
This commit is contained in:
Dongjoon Hyun 2019-04-03 09:41:13 -07:00
parent ffb362a705
commit b51763612a
2 changed files with 0 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
View file

@ -17,13 +17,9 @@
package org.apache.spark.sql.catalyst.optimizer
import scala.collection.mutable
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.util.MapData
import org.apache.spark.sql.types.MapType
/**
* Simplify redundant [[CreateNamedStructLike]], [[CreateArray]] and [[CreateMap]] expressions.
@ -63,16 +59,6 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
Literal(null, ga.dataType)
}
case GetMapValue(CreateMap(elems), key) => CaseKeyWhen(key, elems)
// The case below happens when the map is foldable, but the key is not, so ConstantFolding
// converts the map in a Literal, but the GetMapValue is still there since the key is not
// foldable. It cannot happen in any other case.
case GetMapValue(Literal(map: MapData, MapType(kt, vt, _)), key) if !key.foldable =>
val elems = new mutable.ListBuffer[Literal]
map.foreach(kt, vt, (key, value) => {
elems.append(Literal(key, kt))
elems.append(Literal(value, vt))
})
CaseKeyWhen(key, elems.result())
}
}
}

11
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
View file

@ -452,15 +452,4 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2")
checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](3, 4))), null)
}
test("SPARK-27278: simplify map access with non-foldable key and foldable map") {
val query = relation.select(GetMapValue(CreateMap(Seq(
1L, "a",
2L, "b")), 'id) as "a")
val expected = relation.select(
CaseWhen(Seq(
(EqualTo('id, 1L), "a"),
(EqualTo('id, 2L), "b"))) as "a")
checkRule(query, expected)
}
}