SPARK-1528 - spark on yarn, add support for accessing remote HDFS

Add a config (spark.yarn.access.namenodes) to allow applications running on yarn to access other secure HDFS cluster.  User just specifies the namenodes of the other clusters and we get Tokens for those and ship them with the spark application.

Author: Thomas Graves <tgraves@apache.org>

Closes #1159 from tgravescs/spark-1528 and squashes the following commits:

ddbcd16 [Thomas Graves] review comments
0ac8501 [Thomas Graves] SPARK-1528 - add support for accessing remote HDFS
This commit is contained in:
Thomas Graves 2014-08-05 12:48:26 -05:00
parent e87075df97
commit 2c0f705e26
3 changed files with 101 additions and 17 deletions

View file

@ -106,6 +106,13 @@ Most of the configs are the same for Spark on YARN as for other deployment modes
set this configuration to "hdfs:///some/path".
</td>
</tr>
<tr>
<td><code>spark.yarn.access.namenodes</code></td>
<td>(none)</td>
<td>
A list of secure HDFS namenodes your Spark application is going to access. For example, `spark.yarn.access.namenodes=hdfs://nn1.com:8032,hdfs://nn2.com:8032`. The Spark application must have acess to the namenodes listed and Kerberos must be properly configured to be able to access them (either in the same realm or in a trusted realm). Spark acquires security tokens for each of the namenodes so that the Spark application can access those remote HDFS clusters.
</td>
</tr>
</table>
# Launching Spark on YARN

View file

@ -29,7 +29,7 @@ import org.apache.hadoop.fs._
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.mapred.Master
import org.apache.hadoop.mapreduce.MRJobConfig
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.util.StringUtils
import org.apache.hadoop.yarn.api._
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
@ -191,23 +191,11 @@ trait ClientBase extends Logging {
// Upload Spark and the application JAR to the remote file system if necessary. Add them as
// local resources to the application master.
val fs = FileSystem.get(conf)
val delegTokenRenewer = Master.getMasterPrincipal(conf)
if (UserGroupInformation.isSecurityEnabled()) {
if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
val errorMessage = "Can't get Master Kerberos principal for use as renewer"
logError(errorMessage)
throw new SparkException(errorMessage)
}
}
val dst = new Path(fs.getHomeDirectory(), appStagingDir)
val nns = ClientBase.getNameNodesToAccess(sparkConf) + dst
ClientBase.obtainTokensForNamenodes(nns, conf, credentials)
val replication = sparkConf.getInt("spark.yarn.submit.file.replication", 3).toShort
if (UserGroupInformation.isSecurityEnabled()) {
val dstFs = dst.getFileSystem(conf)
dstFs.addDelegationTokens(delegTokenRenewer, credentials)
}
val localResources = HashMap[String, LocalResource]()
FileSystem.mkdirs(fs, dst, new FsPermission(STAGING_DIR_PERMISSION))
@ -614,4 +602,40 @@ object ClientBase extends Logging {
YarnSparkHadoopUtil.addToEnvironment(env, Environment.CLASSPATH.name, path,
File.pathSeparator)
/**
* Get the list of namenodes the user may access.
*/
private[yarn] def getNameNodesToAccess(sparkConf: SparkConf): Set[Path] = {
sparkConf.get("spark.yarn.access.namenodes", "").split(",").map(_.trim()).filter(!_.isEmpty)
.map(new Path(_)).toSet
}
private[yarn] def getTokenRenewer(conf: Configuration): String = {
val delegTokenRenewer = Master.getMasterPrincipal(conf)
logDebug("delegation token renewer is: " + delegTokenRenewer)
if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
val errorMessage = "Can't get Master Kerberos principal for use as renewer"
logError(errorMessage)
throw new SparkException(errorMessage)
}
delegTokenRenewer
}
/**
* Obtains tokens for the namenodes passed in and adds them to the credentials.
*/
private[yarn] def obtainTokensForNamenodes(paths: Set[Path], conf: Configuration,
creds: Credentials) {
if (UserGroupInformation.isSecurityEnabled()) {
val delegTokenRenewer = getTokenRenewer(conf)
paths.foreach {
dst =>
val dstFs = dst.getFileSystem(conf)
logDebug("getting token for namenode: " + dst)
dstFs.addDelegationTokens(delegTokenRenewer, creds)
}
}
}
}

View file

@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.FunSuite
import org.scalatest.Matchers
@ -38,7 +40,7 @@ import scala.collection.JavaConversions._
import scala.collection.mutable.{ HashMap => MutableHashMap }
import scala.util.Try
import org.apache.spark.SparkConf
import org.apache.spark.{SparkException, SparkConf}
import org.apache.spark.util.Utils
class ClientBaseSuite extends FunSuite with Matchers {
@ -138,6 +140,57 @@ class ClientBaseSuite extends FunSuite with Matchers {
}
}
test("check access nns empty") {
val sparkConf = new SparkConf()
sparkConf.set("spark.yarn.access.namenodes", "")
val nns = ClientBase.getNameNodesToAccess(sparkConf)
nns should be(Set())
}
test("check access nns unset") {
val sparkConf = new SparkConf()
val nns = ClientBase.getNameNodesToAccess(sparkConf)
nns should be(Set())
}
test("check access nns") {
val sparkConf = new SparkConf()
sparkConf.set("spark.yarn.access.namenodes", "hdfs://nn1:8032")
val nns = ClientBase.getNameNodesToAccess(sparkConf)
nns should be(Set(new Path("hdfs://nn1:8032")))
}
test("check access nns space") {
val sparkConf = new SparkConf()
sparkConf.set("spark.yarn.access.namenodes", "hdfs://nn1:8032, ")
val nns = ClientBase.getNameNodesToAccess(sparkConf)
nns should be(Set(new Path("hdfs://nn1:8032")))
}
test("check access two nns") {
val sparkConf = new SparkConf()
sparkConf.set("spark.yarn.access.namenodes", "hdfs://nn1:8032,hdfs://nn2:8032")
val nns = ClientBase.getNameNodesToAccess(sparkConf)
nns should be(Set(new Path("hdfs://nn1:8032"), new Path("hdfs://nn2:8032")))
}
test("check token renewer") {
val hadoopConf = new Configuration()
hadoopConf.set("yarn.resourcemanager.address", "myrm:8033")
hadoopConf.set("yarn.resourcemanager.principal", "yarn/myrm:8032@SPARKTEST.COM")
val renewer = ClientBase.getTokenRenewer(hadoopConf)
renewer should be ("yarn/myrm:8032@SPARKTEST.COM")
}
test("check token renewer default") {
val hadoopConf = new Configuration()
val caught =
intercept[SparkException] {
ClientBase.getTokenRenewer(hadoopConf)
}
assert(caught.getMessage === "Can't get Master Kerberos principal for use as renewer")
}
object Fixtures {
val knownDefYarnAppCP: Seq[String] =