ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Minor changes in comments and strings to address comments in PR 289.

Browse source
This commit is contained in:
Tathagata Das 2013-12-27 12:26:57 -08:00
parent 3618d70b2a
commit 271e3237f3
1 changed files with 6 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
streaming/src/main/scala/org/apache/spark/streaming/dstream/FileInputDStream.scala
View file

@ -99,9 +99,7 @@ class FileInputDStream[K: ClassTag, V: ClassTag, F <: NewInputFormat[K,V] : Clas
}
/**
* Finds files which have modification timestamp <= current time. If some files are being
* deleted in the directory, then it can generate transient exceptions. Hence, multiple
* attempts are made to handle these transient exceptions. Returns 3-tuple
* Find files which have modification timestamp <= current time and return a 3-tuple of
* (new files found, latest modification time among them, files with latest modification time)
*/
private def findNewFiles(currentTime: Long): (Seq[String], Long, Seq[String]) = {
@ -116,9 +114,9 @@ class FileInputDStream[K: ClassTag, V: ClassTag, F <: NewInputFormat[K,V] : Clas
val fileRDDs = files.map(file => context.sparkContext.newAPIHadoopFile[K, V, F](file))
files.zip(fileRDDs).foreach { case (file, rdd) => {
if (rdd.partitions.size == 0) {
logWarning("File " + file + " has no data in it. Are you sure you are following " +
"the move-based method of adding input files? Refer to the programming guide " +
"for more details.")
logError("File " + file + " has no data in it. Spark Streaming can only ingest " +
"files that have been \"moved\" to the directory assigned to the file stream. " +
"Refer to the streaming programming guide for more details.")
}
}}
new UnionRDD(context.sparkContext, fileRDDs)
@ -181,8 +179,8 @@ class FileInputDStream[K: ClassTag, V: ClassTag, F <: NewInputFormat[K,V] : Clas
}
/**
* Custom PathFilter class to find new files that have modification timestamps <= current time, but have not
* been seen before (i.e. the file should not be in lastModTimeFiles)
* Custom PathFilter class to find new files that have modification timestamps <= current time,
* but have not been seen before (i.e. the file should not be in lastModTimeFiles)
*/
private[streaming]
class CustomPathFilter(maxModTime: Long) extends PathFilter {