ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-8141] [SQL] Precompute datatypes for partition columns and reuse it

Browse source 
JIRA: https://issues.apache.org/jira/browse/SPARK-8141

Author: Liang-Chi Hsieh <viirya@gmail.com>

Closes #6687 from viirya/reuse_partition_column_types and squashes the following commits:

dab0688 [Liang-Chi Hsieh] Reuse partitionColumnTypes.
This commit is contained in:
Liang-Chi Hsieh 2015-06-07 15:33:48 +08:00 committed by Cheng Lian
parent 081db9479a
commit 26d07f1ece
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
View file

@ -435,8 +435,9 @@ abstract class HadoopFsRelation private[sql](maybePartitionSpec: Option[Partitio
// partition values.
userDefinedPartitionColumns.map { partitionSchema =>
val spec = discoverPartitions()
val partitionColumnTypes = spec.partitionColumns.map(_.dataType)
val castedPartitions = spec.partitions.map { case p @ Partition(values, path) =>
val literals = values.toSeq.zip(spec.partitionColumns.map(_.dataType)).map {
val literals = values.toSeq.zip(partitionColumnTypes).map {
case (value, dataType) => Literal.create(value, dataType)
}
val castedValues = partitionSchema.zip(literals).map { case (field, literal) =>