#!/bin/bash # Set some sensible defaults export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://$1:8020} function addProperty() { local path=$1 local name=$2 local value=$3 local entry="$name${value}" local escapedEntry=$(echo $entry | sed 's/\//\\\//g') sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path } function configure() { local path=$1 local module=$2 local envPrefix=$3 local var local value echo "Configuring $module" for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` var="${envPrefix}_${c}" value=${!var} echo " - Setting $name=$value" addProperty $HADOOP_HOME/etc/hadoop/$module-site.xml $name "$value" done } configure $HADOOP_HOME/etc/hadoop/core-site.xml core CORE_CONF configure $HADOOP_HOME/etc/hadoop/hdfs-site.xml hdfs HDFS_CONF if [ $AWS_ECS == "true" ] then #echo "$(ip a | grep -A2 eth1 | grep inet | awk '{print $2}' | sed 's#/.*##g' ) $(hostname)" >> /etc/hosts #echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $(hostname)" >> /etc/hosts #if [ "$MASTER_IP" == "127.0.0.1" ]; then #this is master # echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $1" >> /etc/hosts #else # echo "$MASTER_IP $1" >> /etc/hosts #fi if [ "$MASTER_IP" == "127.0.0.1" ]; then #this is master echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $1" >> /etc/hosts #else # NAMENODEIP=$(nslookup $1 | tail -1 | head -1 | awk '{print $3}') # echo "$NAMENODEIP namenode" >> /etc/hosts fi #echo "hosts file: $(cat /etc/hosts)" echo "hostname: $(hostname)" echo "hostname file: $(cat /etc/hostname)" fi #run dnsmasq #ip a #LOCAL_IP=$(ip a | sed -E -n "s/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p") #export DOCKER_IP=$LOCAL_IP #export DOCKER_HOST=$HOSTNAME #echo $DOCKER_HOST #echo $DOCKER_IP #echo "bogus-priv" > /etc/dnsmasq.conf && \ # echo "no-hosts" >> /etc/dnsmasq.conf && \ # echo "keep-in-foreground" >> /etc/dnsmasq.conf && \ # echo "no-resolv" >> /etc/dnsmasq.conf && \ # echo "expand-hosts" >> /etc/dnsmasq.conf && \ # echo "server=8.8.8.8" >> /etc/dnsmasq.conf && \ # echo "server=8.8.4.4" >> /etc/dnsmasq.conf && \ # echo "address=/$HOSTNAME/$LOCAL_IP" >> /etc/dnsmasq.conf && \ # echo "address=/yomama/$LOCAL_IP" >> /etc/dnsmasq.conf && \ # echo "address=/namenode/$LOCAL_IP" >> /etc/dnsmasq.conf #echo $(cat /etc/dnsmasq.conf) #dnsmasq #nslookup $HOSTNAME 127.0.0.1 #drill $HOSTNAME 127.0.0.1 #ping -4 -c 4 yomama #export _JAVA_OPTIONS=$_JAVA_OPTIONS:'-Dsun.net.spi.nameservice.nameservers=127.0.0.1 -Dsun.net.spi.nameservice.provider.1=dns,sun' #export HADOOP_OPTS=$HADOOP_OPTS:'-Dsun.net.spi.nameservice.nameservers=127.0.0.1 -Dsun.net.spi.nameservice.provider.1=dns,sun' #export SPARK_PRINT_LAUNCH_COMMAND=1