docker/spark-docker/base_entry.sh

#!/bin/bash

# Set some sensible defaults
export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://$1:8020}

function addProperty() {
  local path=$1
  local name=$2
  local value=$3

  local entry="<property><name>$name</name><value>${value}</value></property>"
  local escapedEntry=$(echo $entry | sed 's/\//\\\//g')
  sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path
}

function configure() {
    local path=$1
    local module=$2
    local envPrefix=$3

    local var
    local value

    echo "Configuring $module"
    for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do
        name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'`
        var="${envPrefix}_${c}"
        value=${!var}
        echo " - Setting $name=$value"
        addProperty $HADOOP_HOME/etc/hadoop/$module-site.xml $name "$value"
    done
}

configure $HADOOP_HOME/etc/hadoop/core-site.xml core CORE_CONF
configure $HADOOP_HOME/etc/hadoop/hdfs-site.xml hdfs HDFS_CONF

if [ $AWS_ECS == "true" ] 
then
  #echo "$(ip a | grep -A2 eth1 | grep inet | awk '{print $2}' | sed 's#/.*##g' ) $(hostname)" >> /etc/hosts

  #echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $(hostname)" >> /etc/hosts

  #if [ "$MASTER_IP" == "127.0.0.1" ]; then #this is master
  #    echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $1" >> /etc/hosts
  #else
  #    echo "$MASTER_IP $1" >> /etc/hosts
  #fi

  if [ "$MASTER_IP" == "127.0.0.1" ]; 
  then #this is master
      echo "$(ip a | sed -E -n 's/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p') $1" >> /etc/hosts
  #else
  #    NAMENODEIP=$(nslookup $1 | tail -1 | head -1 | awk '{print $3}')
  #    echo "$NAMENODEIP namenode" >> /etc/hosts
  fi

  #echo "hosts file: $(cat /etc/hosts)"
  echo "hostname: $(hostname)"
  echo "hostname file: $(cat /etc/hostname)"

fi

#run dnsmasq
#ip a
#LOCAL_IP=$(ip a | sed -E -n "s/.*inet ([0-9.]+)\/[0-9]+ .* eth0/\1/p")
#export DOCKER_IP=$LOCAL_IP
#export DOCKER_HOST=$HOSTNAME
#echo $DOCKER_HOST
#echo $DOCKER_IP

#echo "bogus-priv" > /etc/dnsmasq.conf && \
#  echo "no-hosts" >> /etc/dnsmasq.conf && \
#  echo "keep-in-foreground" >> /etc/dnsmasq.conf && \
#  echo "no-resolv" >> /etc/dnsmasq.conf && \
#  echo "expand-hosts" >> /etc/dnsmasq.conf && \
#  echo "server=8.8.8.8" >> /etc/dnsmasq.conf && \
#  echo "server=8.8.4.4" >> /etc/dnsmasq.conf && \
#  echo "address=/$HOSTNAME/$LOCAL_IP" >> /etc/dnsmasq.conf && \
#  echo "address=/yomama/$LOCAL_IP" >> /etc/dnsmasq.conf && \
#  echo "address=/namenode/$LOCAL_IP" >> /etc/dnsmasq.conf 
  
#echo $(cat /etc/dnsmasq.conf)

#dnsmasq 

#nslookup $HOSTNAME 127.0.0.1
#drill $HOSTNAME 127.0.0.1

#ping -4 -c 4 yomama

#export _JAVA_OPTIONS=$_JAVA_OPTIONS:'-Dsun.net.spi.nameservice.nameservers=127.0.0.1 -Dsun.net.spi.nameservice.provider.1=dns,sun'
#export HADOOP_OPTS=$HADOOP_OPTS:'-Dsun.net.spi.nameservice.nameservers=127.0.0.1 -Dsun.net.spi.nameservice.provider.1=dns,sun' 
#export SPARK_PRINT_LAUNCH_COMMAND=1