39 lines
878 B
Bash
Executable file
39 lines
878 B
Bash
Executable file
SPARK_IMAGE=$1
|
|
ABS_PATH_CONF=$2
|
|
ABS_PATH_DATA=$3
|
|
docker run -d -v $ABS_PATH_CONF/master:/conf \
|
|
-v $ABS_PATH_DATA:/tmp/data \
|
|
--name namenode \
|
|
-h namenode \
|
|
--network spark-net \
|
|
-p 222:22 \
|
|
-p 4040:4040 \
|
|
-p 6066:6066 \
|
|
-p 7077:7077 \
|
|
-p 8020:8020 \
|
|
-p 8080:8080 \
|
|
-p 50070:50070 \
|
|
--expose 7001 \
|
|
--expose 7002 \
|
|
--expose 7003 \
|
|
--expose 7004 \
|
|
--expose 7005 \
|
|
--expose 7006 \
|
|
--expose 7077 \
|
|
--expose 6066 \
|
|
--expose 4040 \
|
|
--expose 8020 \
|
|
--expose 50070 \
|
|
-e "MASTER=spark://namenode:7077" \
|
|
-e "SPARK_CONF_DIR=/conf" \
|
|
-e "SPARK_PUBLIC_DNS=127.0.0.1" \
|
|
-e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" \
|
|
-e "SPARK_EXECUTOR_MEMORY=8g" \
|
|
-e "SPARK_DAEMON_MEMORY=8g" \
|
|
-e "SPARK_DRIVER_MEMORY=8g" \
|
|
-e "SPARK_WORKER_MEMORY=8g" \
|
|
-e "HDFS_CONF_dfs_client_use_datanode_hostname=true" \
|
|
-e "AWS_ECS=false" \
|
|
$SPARK_IMAGE \
|
|
/usr/local/spark-2.2.0-bin-without-hadoop/master.sh
|