49 lines
6.1 KiB
Bash
49 lines
6.1 KiB
Bash
|
|
||
|
|
|
||
|
|
sudo docker run -d -v ~/conf/master:/conf -v ~/data:/tmp/data --name namenode -h namenode --network spark-net -p 222:22 -p 4040:4040 -p 6066:6066 -p 7077:7077 -p 8020:8020 -p 8080:8080 -p 50070:50070 --expose 7001 --expose 7002 --expose 7003 --expose 7004 --expose 7005 --expose 7006 --expose 7077 --expose 6066 --expose 4040 --expose 8020 --expose 50070 -e "MASTER=spark://namenode:7077" -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/master.sh
|
||
|
|
|
||
|
|
MASTER_CONTAINER=354887502f6c35be579f80818cea6654a80a276c4a54ad450d46aa8de2f8937f
|
||
|
|
START_PORT=7022
|
||
|
|
END_PORT=7026
|
||
|
|
WORKER_PORT=8882
|
||
|
|
WORKER_WEBUI_PORT=8082
|
||
|
|
HOSTNAME="datanode"
|
||
|
|
DATANODE_PORT=50010
|
||
|
|
sudo docker run -d -v ~/conf/worker:/conf -v ~/data:/tmp/data -h $HOSTNAME --name $HOSTNAME --network spark-net --link $MASTER_CONTAINER -p $WORKER_WEBUI_PORT:$WORKER_WEBUI_PORT -p $DATANODE_PORT:$DATANODE_PORT --expose 30000-50000 --expose $START_PORT-$END_PORT --expose $WORKER_PORT --expose $DATANODE_PORT -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "SPARK_WORKER_CORES=4" -e "SPARK_WORKER_PORT=$WORKER_PORT" -e "SPARK_WORKER_WEBUI_PORT=$WORKER_WEBUI_PORT" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "HDFS_DATA_HOST=$HOSTNAME" -e "HDFS_HOST=namenode" -e "HDFS_CONF_dfs_datanode_address=0.0.0.0:$DATANODE_PORT" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/worker.sh
|
||
|
|
|
||
|
|
START_PORT=7032
|
||
|
|
END_PORT=7036
|
||
|
|
WORKER_PORT=8883
|
||
|
|
WORKER_WEBUI_PORT=8083
|
||
|
|
HOSTNAME="datanode2"
|
||
|
|
DATANODE_PORT=50011
|
||
|
|
sudo docker run -d -v ~/conf/worker:/conf -v ~/data:/tmp/data -h $HOSTNAME --name $HOSTNAME --network spark-net --link $MASTER_CONTAINER -p $WORKER_WEBUI_PORT:$WORKER_WEBUI_PORT -p $DATANODE_PORT:$DATANODE_PORT --expose 30000-50000 --expose $START_PORT-$END_PORT --expose $WORKER_PORT --expose $DATANODE_PORT -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "SPARK_WORKER_CORES=4" -e "SPARK_WORKER_PORT=$WORKER_PORT" -e "SPARK_WORKER_WEBUI_PORT=$WORKER_WEBUI_PORT" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "HDFS_DATA_HOST=$HOSTNAME" -e "HDFS_HOST=namenode" -e "HDFS_CONF_dfs_datanode_address=0.0.0.0:$DATANODE_PORT" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/worker.sh
|
||
|
|
|
||
|
|
START_PORT=7042
|
||
|
|
END_PORT=7046
|
||
|
|
WORKER_PORT=8884
|
||
|
|
WORKER_WEBUI_PORT=8084
|
||
|
|
HOSTNAME="datanode3"
|
||
|
|
DATANODE_PORT=50012
|
||
|
|
sudo docker run -d -v ~/conf/worker:/conf -v ~/data:/tmp/data -h $HOSTNAME --name $HOSTNAME --network spark-net --link $MASTER_CONTAINER -p $WORKER_WEBUI_PORT:$WORKER_WEBUI_PORT -p $DATANODE_PORT:$DATANODE_PORT --expose 30000-50000 --expose $START_PORT-$END_PORT --expose $WORKER_PORT --expose $DATANODE_PORT -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "SPARK_WORKER_CORES=4" -e "SPARK_WORKER_PORT=$WORKER_PORT" -e "SPARK_WORKER_WEBUI_PORT=$WORKER_WEBUI_PORT" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "HDFS_DATA_HOST=$HOSTNAME" -e "HDFS_HOST=namenode" -e "HDFS_CONF_dfs_datanode_address=0.0.0.0:$DATANODE_PORT" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/worker.sh
|
||
|
|
|
||
|
|
START_PORT=7052
|
||
|
|
END_PORT=7056
|
||
|
|
WORKER_PORT=8885
|
||
|
|
WORKER_WEBUI_PORT=8085
|
||
|
|
HOSTNAME="datanode4"
|
||
|
|
DATANODE_PORT=50013
|
||
|
|
sudo docker run -d -v ~/conf/worker:/conf -v ~/data:/tmp/data -h $HOSTNAME --name $HOSTNAME --network spark-net --link $MASTER_CONTAINER -p $WORKER_WEBUI_PORT:$WORKER_WEBUI_PORT -p $DATANODE_PORT:$DATANODE_PORT --expose 30000-50000 --expose $START_PORT-$END_PORT --expose $WORKER_PORT --expose $DATANODE_PORT -e "SPARK_CONF_DIR=/conf" -e "SPARK_PUBLIC_DNS=127.0.0.1" -e "SPARK_WORKER_CORES=4" -e "SPARK_WORKER_PORT=$WORKER_PORT" -e "SPARK_WORKER_WEBUI_PORT=$WORKER_WEBUI_PORT" -e "LD_LIBRARY_PATH=/usr/local/hadoop/lib/native/" -e "HDFS_DATA_HOST=$HOSTNAME" -e "HDFS_HOST=namenode" -e "HDFS_CONF_dfs_datanode_address=0.0.0.0:$DATANODE_PORT" -e "SPARK_EXECUTOR_MEMORY=8g" -e "SPARK_DAEMON_MEMORY=8g" -e "SPARK_DRIVER_MEMORY=8g" -e "SPARK_WORKER_MEMORY=8g" -e "HDFS_CONF_dfs_client_use_datanode_hostname=true" -e "AWS_ECS=false" docker.mimirdb.info/spark-hadoop /usr/local/spark-2.2.0-bin-without-hadoop/worker.sh
|
||
|
|
|
||
|
|
|
||
|
|
#mimir
|
||
|
|
sudo docker run -d --mount source=vizier-data-ub,destination=/usr/local/source/web-api/.vizierdb -p 9002:9001 -p 33388:33388 --network spark-net -h vizier-mimir --name vizier-mimir -e RESTORE_BACKUP=false -e USE_S3_VOLUME=false -e PULL_MIMIR=false -e AWS_ACCESS_KEY_ID=AKIAJ7MLFSPYLYG47ARQ -e AWS_SECRET_ACCESS_KEY=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 -e S3_BUCKET_NAME="vizier-data-ub" --privileged --device /dev/fuse docker.mimirdb.info/vizier-mimir-spark
|
||
|
|
|
||
|
|
#api
|
||
|
|
sudo docker run -d --mount source=vizier-data-ub,destination=/usr/local/source/web-api/.vizierdb -p 9003:9001 -p 81:80 -p 5000:443 --network spark-net -h vizier-api --name vizier-api -e MIMIR_HOST="vizier-mimir" -e USE_S3_VOLUME=false -e APP_PATH="" -e API_SERVER=api.vizierdb.info -e API_LOCAL_PORT=443 -e API_PORT=443 -e API_SCHEME=https -e AWS_ACCESS_KEY_ID=AKIAJ7MLFSPYLYG47ARQ -e AWS_SECRET_ACCESS_KEY=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 -e S3_BUCKET_NAME="vizier-data-ub" --privileged --device /dev/fuse docker.mimirdb.info/vizier-api-spark
|
||
|
|
|
||
|
|
#ui
|
||
|
|
sudo docker run -d -e API_SERVER=api.vizierdb.info -e API_PORT=443 -e API_SCHEME=https -p 3000:80 -p 9004:9001 -h vizier-ui --name vizier-ui --network spark-net docker.mimirdb.info/vizier-ui
|
||
|
|
|
||
|
|
|
||
|
|
runBackup --restore --sparkHost namenode --dataStagingType s3 --overwriteJars -X LOG LOGM remoteSpark
|