From e47f48c737052564e92903de16ff16707fae32c3 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 30 Aug 2017 13:03:30 +0900 Subject: [PATCH] [SPARK-20886][CORE] HadoopMapReduceCommitProtocol to handle FileOutputCommitter.getWorkPath==null ## What changes were proposed in this pull request? Handles the situation where a `FileOutputCommitter.getWorkPath()` returns `null` by downgrading to the supplied `path` argument. The existing code does an `Option(workPath.toString).getOrElse(path)`, which triggers an NPE in the `toString()` operation if the workPath == null. The code apparently was meant to handle this (hence the getOrElse() clause, but as the NPE has already occurred at that point the else-clause never gets invoked. ## How was this patch tested? Manually, with some later code review. Author: Steve Loughran Closes #18111 from steveloughran/cloud/SPARK-20886-committer-NPE. --- .../spark/internal/io/HadoopMapReduceCommitProtocol.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala index 22e2679913..b1d07ab2c9 100644 --- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala +++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala @@ -73,7 +73,8 @@ class HadoopMapReduceCommitProtocol(jobId: String, path: String) val stagingDir: String = committer match { // For FileOutputCommitter it has its own staging path called "work path". - case f: FileOutputCommitter => Option(f.getWorkPath.toString).getOrElse(path) + case f: FileOutputCommitter => + Option(f.getWorkPath).map(_.toString).getOrElse(path) case _ => path }