From b51763612a71e214ed94ac5fb6843cf03d00f9f8 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 3 Apr 2019 09:41:13 -0700 Subject: [PATCH] Revert "[SPARK-27278][SQL] Optimize GetMapValue when the map is a foldable and the key is not" This reverts commit 5888b15d9cdf8272012018f39bf58c8faf68a5e1. --- .../sql/catalyst/optimizer/ComplexTypes.scala | 14 -------------- .../sql/catalyst/optimizer/complexTypesSuite.scala | 11 ----------- 2 files changed, 25 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala index 0255128138..db7d6d3254 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala @@ -17,13 +17,9 @@ package org.apache.spark.sql.catalyst.optimizer -import scala.collection.mutable - import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.MapData -import org.apache.spark.sql.types.MapType /** * Simplify redundant [[CreateNamedStructLike]], [[CreateArray]] and [[CreateMap]] expressions. @@ -63,16 +59,6 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] { Literal(null, ga.dataType) } case GetMapValue(CreateMap(elems), key) => CaseKeyWhen(key, elems) - // The case below happens when the map is foldable, but the key is not, so ConstantFolding - // converts the map in a Literal, but the GetMapValue is still there since the key is not - // foldable. It cannot happen in any other case. - case GetMapValue(Literal(map: MapData, MapType(kt, vt, _)), key) if !key.foldable => - val elems = new mutable.ListBuffer[Literal] - map.foreach(kt, vt, (key, value) => { - elems.append(Literal(key, kt)) - elems.append(Literal(value, vt)) - }) - CaseKeyWhen(key, elems.result()) } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala index 84c74a693f..5452e72b38 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala @@ -452,15 +452,4 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper { checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2") checkEvaluation(GetMapValue(mb0, Literal(Array[Byte](3, 4))), null) } - - test("SPARK-27278: simplify map access with non-foldable key and foldable map") { - val query = relation.select(GetMapValue(CreateMap(Seq( - 1L, "a", - 2L, "b")), 'id) as "a") - val expected = relation.select( - CaseWhen(Seq( - (EqualTo('id, 1L), "a"), - (EqualTo('id, 2L), "b"))) as "a") - checkRule(query, expected) - } }