ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Check for AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables.

Browse source 
For custom properties, use "spark.hadoop.*" as a prefix instead of just "hadoop.*".
This commit is contained in:
Stephen Haberman 2013-01-10 10:55:41 -06:00
parent e3861ae395
commit b15e851279
1 changed files with 10 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
core/src/main/scala/spark/SparkContext.scala
View file

@ -190,9 +190,16 @@ class SparkContext(
/** A default Hadoop Configuration for the Hadoop code (e.g. file systems) that we reuse. */ /** A default Hadoop Configuration for the Hadoop code (e.g. file systems) that we reuse. */
val hadoopConfiguration = { val hadoopConfiguration = {
val conf = new Configuration() val conf = new Configuration()
// Copy any "hadoop.foo=bar" system properties into conf as "foo=bar" // Explicitly check for S3 environment variables
for (key <- System.getProperties.keys.asInstanceOf[Set[String]] if key.startsWith("hadoop.")) { if (System.getenv("AWS_ACCESS_KEY_ID") != null && System.getenv("AWS_SECRET_ACCESS_KEY") != null) {
conf.set(key.substring("hadoop.".length), System.getProperty(key)) conf.set("fs.s3.awsAccessKeyId", System.getenv("AWS_ACCESS_KEY_ID"))
conf.set("fs.s3n.awsAccessKeyId", System.getenv("AWS_ACCESS_KEY_ID"))
conf.set("fs.s3.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY"))
conf.set("fs.s3n.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY"))
}
// Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar"
for (key <- System.getProperties.keys.asInstanceOf[Set[String]] if key.startsWith("spark.hadoop.")) {
conf.set(key.substring("spark.hadoop.".length), System.getProperty(key))
} }
val bufferSize = System.getProperty("spark.buffer.size", "65536") val bufferSize = System.getProperty("spark.buffer.size", "65536")
conf.set("io.file.buffer.size", bufferSize) conf.set("io.file.buffer.size", bufferSize)