ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Revert "[SPARK-8437] [DOCS] Using directory path without wildcard for filename slow for large number of files with wholeTextFiles and binaryFiles"

Browse source 
This reverts commit b2684557fa.
This commit is contained in:
Andrew Or 2015-06-29 18:32:59 -07:00
parent c0fbd6781e
commit 80d53565ae
1 changed files with 2 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
core/src/main/scala/org/apache/spark/SparkContext.scala
View file

@ -824,8 +824,6 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* }}} * }}}
* *
* @note Small files are preferred, large file is also allowable, but may cause bad performance. * @note Small files are preferred, large file is also allowable, but may cause bad performance.
* @note On some filesystems, `.../path/*` can be a more efficient way to read all files in a directory
* rather than `.../path/` or `.../path`
* *
* @param minPartitions A suggestion value of the minimal splitting number for input data. * @param minPartitions A suggestion value of the minimal splitting number for input data.
*/ */
@ -873,11 +871,9 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* (a-hdfs-path/part-nnnnn, its content) * (a-hdfs-path/part-nnnnn, its content)
* }}} * }}}
* *
* @note Small files are preferred; very large files may cause bad performance.
* @note On some filesystems, `.../path/*` can be a more efficient way to read all files in a directory
* rather than `.../path/` or `.../path`
*
* @param minPartitions A suggestion value of the minimal splitting number for input data. * @param minPartitions A suggestion value of the minimal splitting number for input data.
*
* @note Small files are preferred; very large files may cause bad performance.
*/ */
@Experimental @Experimental
def binaryFiles( def binaryFiles(