[SPARK-4606] Send EOF to child JVM when there's no more data to read.
Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #3460 from vanzin/SPARK-4606 and squashes the following commits: 031207d [Marcelo Vanzin] [SPARK-4606] Send EOF to child JVM when there's no more data to read.
This commit is contained in:
parent
3f5f4cc4e7
commit
7e2deb71c4
|
@ -151,7 +151,8 @@ private[spark] object SparkSubmitDriverBootstrapper {
|
|||
val isWindows = Utils.isWindows
|
||||
val isSubprocess = sys.env.contains("IS_SUBPROCESS")
|
||||
if (!isWindows) {
|
||||
val stdinThread = new RedirectThread(System.in, process.getOutputStream, "redirect stdin")
|
||||
val stdinThread = new RedirectThread(System.in, process.getOutputStream, "redirect stdin",
|
||||
propagateEof = true)
|
||||
stdinThread.start()
|
||||
// Spark submit (JVM) may run as a subprocess, and so this JVM should terminate on
|
||||
// broken pipe, signaling that the parent process has exited. This is the case if the
|
||||
|
|
|
@ -1847,13 +1847,18 @@ private[spark] object Utils extends Logging {
|
|||
/**
|
||||
* A utility class to redirect the child process's stdout or stderr.
|
||||
*/
|
||||
private[spark] class RedirectThread(in: InputStream, out: OutputStream, name: String)
|
||||
private[spark] class RedirectThread(
|
||||
in: InputStream,
|
||||
out: OutputStream,
|
||||
name: String,
|
||||
propagateEof: Boolean = false)
|
||||
extends Thread(name) {
|
||||
|
||||
setDaemon(true)
|
||||
override def run() {
|
||||
scala.util.control.Exception.ignoring(classOf[IOException]) {
|
||||
// FIXME: We copy the stream on the level of bytes to avoid encoding problems.
|
||||
try {
|
||||
val buf = new Array[Byte](1024)
|
||||
var len = in.read(buf)
|
||||
while (len != -1) {
|
||||
|
@ -1861,6 +1866,11 @@ private[spark] class RedirectThread(in: InputStream, out: OutputStream, name: St
|
|||
out.flush()
|
||||
len = in.read(buf)
|
||||
}
|
||||
} finally {
|
||||
if (propagateEof) {
|
||||
out.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue