[SPARK-3674] YARN support in Spark EC2
This corresponds to https://github.com/mesos/spark-ec2/pull/116 in the spark-ec2 repo. The only changes required on the spark_ec2.py script is to open the RM port.
cc andrewor14
Author: Shivaram Venkataraman <shivaram@cs.berkeley.edu>
Closes #6376 from shivaram/spark-ec2-yarn and squashes the following commits:
961504a [Shivaram Venkataraman] Merge branch 'master' of https://github.com/apache/spark into spark-ec2-yarn
152c94c [Shivaram Venkataraman] Open 8088 for YARN in EC2
(cherry picked from commit 2e9a5f229e
)
Signed-off-by: Andrew Or <andrew@databricks.com>
This commit is contained in:
parent
51d98b0e97
commit
d014a447a3
|
@ -492,6 +492,8 @@ def launch_cluster(conn, opts, cluster_name):
|
|||
master_group.authorize('udp', 2049, 2049, authorized_address)
|
||||
master_group.authorize('tcp', 4242, 4242, authorized_address)
|
||||
master_group.authorize('udp', 4242, 4242, authorized_address)
|
||||
# RM in YARN mode uses 8088
|
||||
master_group.authorize('tcp', 8088, 8088, authorized_address)
|
||||
if opts.ganglia:
|
||||
master_group.authorize('tcp', 5080, 5080, authorized_address)
|
||||
if slave_group.rules == []: # Group was just now created
|
||||
|
|
Loading…
Reference in a new issue