2ddb6f262f
-there is one issue with the mimir hostname not being resolved from spark workers. may switch to stateful setfor mimir to resolve.
31 lines
3.1 KiB
Bash
31 lines
3.1 KiB
Bash
|
|
#run the containers
|
|
#spark-master
|
|
cd ../spark-docker
|
|
MASTER_CONTAINER=`sudo docker run -d -v conf/master:/conf -v data:/tmp/data --name namenode -h namenode --network spark-net -p 222:22 -p 4040:4040 -p 6066:6066 -p 7077:7077 -p 8020:8020 -p 8080:8080 -p 50070:50070 --expose 7001 --expose 7002 --expose 7003 --expose 7004 --expose 7005 --expose 7006 --expose 7077 --expose 6066 --expose 4040 --expose 8020 --expose 50070 -e "MASTER=spark://namenode:7077" -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/master.sh`
|
|
|
|
#spark-worker
|
|
cd ../spark-docker
|
|
START_PORT=7022
|
|
END_PORT=7026
|
|
WORKER_PORT=8882
|
|
WORKER_WEBUI_PORT=8082
|
|
HOSTNAME="datanode"
|
|
DATANODE_PORT=50010
|
|
sudo docker run -d -v conf/worker:/conf -v data:/tmp/data -h $HOSTNAME --name $HOSTNAME --network spark-net --link $MASTER_CONTAINER -p $WORKER_WEBUI_PORT:$WORKER_WEBUI_PORT -p $DATANODE_PORT:$DATANODE_PORT --expose 30000-50000 --expose $START_PORT-$END_PORT --expose $WORKER_PORT --expose $DATANODE_PORT -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "SPARK_WORKER_CORES=4" -e "SPARK_WORKER_PORT=$WORKER_PORT" -e "SPARK_WORKER_WEBUI_PORT=$WORKER_WEBUI_PORT" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "HDFS_DATA_HOST=$HOSTNAME" -e "HDFS_HOST=spark-master" -e "HDFS_CONF_dfs_datanode_address=0.0.0.0:$DATANODE_PORT" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/worker.sh
|
|
|
|
#mimir
|
|
sudo docker run -d -p 9002:9001 -p 33388:33388 --network spark-net -h vizier-mimir --name vizier-mimir -e RESTORE_BACKUP=false -e PULL_MIMIR=false -e AWS_ACCESS_KEY_ID=AKIAJ7MLFSPYLYG47ARQ -e AWS_SECRET_ACCESS_KEY=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 -e S3_BUCKET_NAME="vizier-data-test" --privileged --device /dev/fuse docker.mimirdb.info/vizier-mimir-spark
|
|
|
|
#api
|
|
sudo docker run -d -p 9003:9001 -p 81:80 -p 444:443 --network spark-net -h vizier-api --name vizier-api -e MIMIR_HOST="vizier-mimir" -e APP_PATH="" -e API_SERVER=localhost -e API_LOCAL_PORT=443 -e API_PORT=444 -e API_SCHEME=http -e AWS_ACCESS_KEY_ID=AKIAJ7MLFSPYLYG47ARQ -e AWS_SECRET_ACCESS_KEY=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 -e S3_BUCKET_NAME="vizier-data-test" --privileged --device /dev/fuse docker.mimirdb.info/vizier-api-spark
|
|
|
|
#ui
|
|
sudo docker run -d -e API_SERVER=localhost -e API_PORT=444 -e API_SCHEME=http -p 80:80 -p 443:443 -p 9004:9001 -h vizier-ui --name vizier-ui --network spark-net docker.mimirdb.info/vizier-ui
|
|
|
|
#analytics
|
|
sudo docker run -d -p 80:80 -p 9001:9001 -h vizier-analytics --network spark-net docker.mimirdb.info/vizier-analytics
|
|
|
|
#proxy
|
|
sudo docker run -d -p 80:80 -p 443:443 -p 9001:9001 -h vizier-proxy --network spark-net docker.mimirdb.info/vizier-proxy
|