ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-14208][SQL] Renames spark.sql.parquet.fileScan

Browse source 
## What changes were proposed in this pull request?

Renames SQL option `spark.sql.parquet.fileScan` since now all `HadoopFsRelation` based data sources are being migrated to `FileScanRDD` code path.

## How was this patch tested?

None.

Author: Cheng Lian <lian@databricks.com>

Closes #12003 from liancheng/spark-14208-option-renaming.
This commit is contained in:
Cheng Lian 2016-03-29 20:56:01 +08:00
parent 425bcf6d68
commit a632bb56f8
2 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
View file

@ -60,7 +60,7 @@ private[sql] object FileSourceStrategy extends Strategy with Logging {
files.fileFormat.isInstanceOf[parquet.DefaultSource] ||
files.fileFormat.toString == "ORC" ||
files.fileFormat.isInstanceOf[json.DefaultSource]) &&
files.sqlContext.conf.parquetFileScan =>
files.sqlContext.conf.useFileScan =>
// Filters on this relation fall into four categories based on where we can use them to avoid
// reading unneeded data:
// - partition keys only - used to prune directories to read

8
sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file

@ -288,9 +288,9 @@ object SQLConf {
defaultValue = Some(true),
doc = "Whether the query analyzer should be case sensitive or not.")
val PARQUET_FILE_SCAN = booleanConf("spark.sql.parquet.fileScan",
val USE_FILE_SCAN = booleanConf("spark.sql.sources.fileScan",
defaultValue = Some(true),
doc = "Use the new FileScanRDD path for reading parquet data.",
doc = "Use the new FileScanRDD path for reading HDSF based data sources.",
isPublic = false)
val PARQUET_SCHEMA_MERGING_ENABLED = booleanConf("spark.sql.parquet.mergeSchema",
@ -583,9 +583,9 @@ class SQLConf extends Serializable with CatalystConf with ParserConf with Loggin
def useCompression: Boolean = getConf(COMPRESS_CACHED)
def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION)
def useFileScan: Boolean = getConf(USE_FILE_SCAN)
def parquetFileScan: Boolean = getConf(PARQUET_FILE_SCAN)
def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION)
def parquetCacheMetadata: Boolean = getConf(PARQUET_CACHE_METADATA)