57adb77e6b
This helps with preventing jdk8-specific calls being checked in, because PR builders are running the compiler with the wrong settings. If the JAVA_7_HOME env variable is set, assume it points at a jdk7 and use its rt.jar when invoking javac. For zinc, just run it with jdk7, and disable it when building jdk8-specific code. A big note for sbt usage: adding the bootstrap options forces sbt to fork the compiler, and that disables incremental compilation. That means that it's really not convenient to use for normal development, but should be ok for automated builds. Tested with JAVA_HOME=jdk8 and JAVA_7_HOME=jdk7: - mvn + zinc - mvn sans zinc - sbt Verified that in all cases, jdk8-specific library calls fail to compile. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #13272 from vanzin/SPARK-15451.
162 lines
5.9 KiB
Bash
Executable file
162 lines
5.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Determine the current working directory
|
|
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
# Preserve the calling directory
|
|
_CALLING_DIR="$(pwd)"
|
|
# Options used during compilation
|
|
_COMPILE_JVM_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
|
|
|
|
# Installs any application tarball given a URL, the expected tarball name,
|
|
# and, optionally, a checkable binary path to determine if the binary has
|
|
# already been installed
|
|
## Arg1 - URL
|
|
## Arg2 - Tarball Name
|
|
## Arg3 - Checkable Binary
|
|
install_app() {
|
|
local remote_tarball="$1/$2"
|
|
local local_tarball="${_DIR}/$2"
|
|
local binary="${_DIR}/$3"
|
|
|
|
# setup `curl` and `wget` silent options if we're running on Jenkins
|
|
local curl_opts="-L"
|
|
local wget_opts=""
|
|
if [ -n "$AMPLAB_JENKINS" ]; then
|
|
curl_opts="-s ${curl_opts}"
|
|
wget_opts="--quiet ${wget_opts}"
|
|
else
|
|
curl_opts="--progress-bar ${curl_opts}"
|
|
wget_opts="--progress=bar:force ${wget_opts}"
|
|
fi
|
|
|
|
if [ -z "$3" -o ! -f "$binary" ]; then
|
|
# check if we already have the tarball
|
|
# check if we have curl installed
|
|
# download application
|
|
[ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
|
|
echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
|
|
curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
|
|
# if the file still doesn't exist, lets try `wget` and cross our fingers
|
|
[ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
|
|
echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
|
|
wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
|
|
# if both were unsuccessful, exit
|
|
[ ! -f "${local_tarball}" ] && \
|
|
echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
|
|
echo "please install manually and try again." && \
|
|
exit 2
|
|
cd "${_DIR}" && tar -xzf "$2"
|
|
rm -rf "$local_tarball"
|
|
fi
|
|
}
|
|
|
|
# Determine the Maven version from the root pom.xml file and
|
|
# install maven under the build/ folder if needed.
|
|
install_mvn() {
|
|
local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
|
|
MVN_BIN="$(command -v mvn)"
|
|
if [ "$MVN_BIN" ]; then
|
|
local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
|
|
fi
|
|
# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
|
|
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
|
|
if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
|
|
local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
|
|
|
|
install_app \
|
|
"${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
|
|
"apache-maven-${MVN_VERSION}-bin.tar.gz" \
|
|
"apache-maven-${MVN_VERSION}/bin/mvn"
|
|
|
|
MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
|
|
fi
|
|
}
|
|
|
|
# Install zinc under the build/ folder
|
|
install_zinc() {
|
|
local zinc_path="zinc-0.3.9/bin/zinc"
|
|
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
|
|
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com}
|
|
|
|
install_app \
|
|
"${TYPESAFE_MIRROR}/zinc/0.3.9" \
|
|
"zinc-0.3.9.tgz" \
|
|
"${zinc_path}"
|
|
ZINC_BIN="${_DIR}/${zinc_path}"
|
|
}
|
|
|
|
# Determine the Scala version from the root pom.xml file, set the Scala URL,
|
|
# and, with that, download the specific version of Scala necessary under
|
|
# the build/ folder
|
|
install_scala() {
|
|
# determine the Scala version used in Spark
|
|
local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
|
|
local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
|
|
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com}
|
|
|
|
install_app \
|
|
"${TYPESAFE_MIRROR}/scala/${scala_version}" \
|
|
"scala-${scala_version}.tgz" \
|
|
"scala-${scala_version}/bin/scala"
|
|
|
|
SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
|
|
SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
|
|
}
|
|
|
|
# Setup healthy defaults for the Zinc port if none were provided from
|
|
# the environment
|
|
ZINC_PORT=${ZINC_PORT:-"3030"}
|
|
|
|
# Remove `--force` for backward compatibility.
|
|
if [ "$1" == "--force" ]; then
|
|
echo "WARNING: '--force' is deprecated and ignored."
|
|
shift
|
|
fi
|
|
|
|
# Install the proper version of Scala, Zinc and Maven for the build
|
|
install_zinc
|
|
install_scala
|
|
install_mvn
|
|
|
|
# Reset the current working directory
|
|
cd "${_CALLING_DIR}"
|
|
|
|
# Now that zinc is ensured to be installed, check its status and, if its
|
|
# not running or just installed, start it
|
|
if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
|
|
ZINC_JAVA_HOME=
|
|
if [ -n "$JAVA_7_HOME" ]; then
|
|
ZINC_JAVA_HOME="env JAVA_HOME=$JAVA_7_HOME"
|
|
fi
|
|
export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
|
|
"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
|
|
$ZINC_JAVA_HOME "${ZINC_BIN}" -start -port ${ZINC_PORT} \
|
|
-scala-compiler "${SCALA_COMPILER}" \
|
|
-scala-library "${SCALA_LIBRARY}" &>/dev/null
|
|
fi
|
|
|
|
# Set any `mvn` options if not already present
|
|
export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
|
|
|
|
echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
|
|
|
|
# Last, call the `mvn` command as usual
|
|
${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
|