ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-7838] [STREAMING] Set scope for kinesis stream

Browse source 
Author: Tathagata Das <tathagata.das1565@gmail.com>

Closes #6369 from tdas/SPARK-7838 and squashes the following commits:

87d1c7f [Tathagata Das] Addressed comment
37775d8 [Tathagata Das] set scope for kinesis stream
This commit is contained in:
Tathagata Das 2015-05-22 23:05:54 -07:00 committed by Andrew Or
parent 017b3404a5
commit baa89838cc
2 changed files with 7 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala
View file

@ -63,9 +63,12 @@ object KinesisUtils {
checkpointInterval: Duration, checkpointInterval: Duration,
storageLevel: StorageLevel storageLevel: StorageLevel
): ReceiverInputDStream[Array[Byte]] = { ): ReceiverInputDStream[Array[Byte]] = {
ssc.receiverStream( // Setting scope to override receiver stream's scope of "receiver stream"
new KinesisReceiver(kinesisAppName, streamName, endpointUrl, validateRegion(regionName), ssc.withNamedScope("kinesis stream") {
initialPositionInStream, checkpointInterval, storageLevel, None)) ssc.receiverStream(
new KinesisReceiver(kinesisAppName, streamName, endpointUrl, validateRegion(regionName),
initialPositionInStream, checkpointInterval, storageLevel, None))
}
} }
/** /**

2
streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
View file

@ -262,7 +262,7 @@ class StreamingContext private[streaming] (
* *
* Note: Return statements are NOT allowed in the given body. * Note: Return statements are NOT allowed in the given body.
*/ */
private def withNamedScope[U](name: String)(body: => U): U = { private[streaming] def withNamedScope[U](name: String)(body: => U): U = {
RDDOperationScope.withScope(sc, name, allowNesting = false, ignoreParent = false)(body) RDDOperationScope.withScope(sc, name, allowNesting = false, ignoreParent = false)(body)
} }