Check for AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables.
For custom properties, use "spark.hadoop.*" as a prefix instead of just "hadoop.*".
This commit is contained in:
parent
e3861ae395
commit
b15e851279
|
@ -190,9 +190,16 @@ class SparkContext(
|
||||||
/** A default Hadoop Configuration for the Hadoop code (e.g. file systems) that we reuse. */
|
/** A default Hadoop Configuration for the Hadoop code (e.g. file systems) that we reuse. */
|
||||||
val hadoopConfiguration = {
|
val hadoopConfiguration = {
|
||||||
val conf = new Configuration()
|
val conf = new Configuration()
|
||||||
// Copy any "hadoop.foo=bar" system properties into conf as "foo=bar"
|
// Explicitly check for S3 environment variables
|
||||||
for (key <- System.getProperties.keys.asInstanceOf[Set[String]] if key.startsWith("hadoop.")) {
|
if (System.getenv("AWS_ACCESS_KEY_ID") != null && System.getenv("AWS_SECRET_ACCESS_KEY") != null) {
|
||||||
conf.set(key.substring("hadoop.".length), System.getProperty(key))
|
conf.set("fs.s3.awsAccessKeyId", System.getenv("AWS_ACCESS_KEY_ID"))
|
||||||
|
conf.set("fs.s3n.awsAccessKeyId", System.getenv("AWS_ACCESS_KEY_ID"))
|
||||||
|
conf.set("fs.s3.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY"))
|
||||||
|
conf.set("fs.s3n.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY"))
|
||||||
|
}
|
||||||
|
// Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar"
|
||||||
|
for (key <- System.getProperties.keys.asInstanceOf[Set[String]] if key.startsWith("spark.hadoop.")) {
|
||||||
|
conf.set(key.substring("spark.hadoop.".length), System.getProperty(key))
|
||||||
}
|
}
|
||||||
val bufferSize = System.getProperty("spark.buffer.size", "65536")
|
val bufferSize = System.getProperty("spark.buffer.size", "65536")
|
||||||
conf.set("io.file.buffer.size", bufferSize)
|
conf.set("io.file.buffer.size", bufferSize)
|
||||||
|
|
Loading…
Reference in a new issue