3c3533d845
### What changes were proposed in this pull request? change $(command -v curl) to "$(command -v curl)" ### Why are the changes needed? We need change $(command -v curl) to "$(command -v curl)" to make sure it work when `curl` or `wget` is uninstall. othewise raised: `build/mvn: line 56: [: /root/spark/build/apache-maven-3.6.3-bin.tar.gz: binary operator expected` ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? ``` apt remove curl rm -f build/apache-maven-3.6.3-bin.tar.gz rm -r build/apache-maven-3.6.3-bin mvn -v ``` Closes #32608 from Yikun/patch-6. Authored-by: Yikun Jiang <yikunkero@gmail.com> Signed-off-by: Sean Owen <srowen@gmail.com>
171 lines
6.3 KiB
Bash
Executable file
171 lines
6.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Determine the current working directory
|
|
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
# Preserve the calling directory
|
|
_CALLING_DIR="$(pwd)"
|
|
# Options used during compilation
|
|
_COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"
|
|
|
|
# Installs any application tarball given a URL, the expected tarball name,
|
|
# and, optionally, a checkable binary path to determine if the binary has
|
|
# already been installed. Arguments:
|
|
# 1 - Mirror host
|
|
# 2 - URL path on host
|
|
# 3 - URL query string
|
|
# 4 - checksum suffix
|
|
# 5 - Tarball Name
|
|
# 6 - Checkable Binary
|
|
install_app() {
|
|
local mirror_host="$1"
|
|
local url_path="$2"
|
|
local url_query="$3"
|
|
local checksum_suffix="$4"
|
|
local local_tarball="${_DIR}/$5"
|
|
local binary="${_DIR}/$6"
|
|
local remote_tarball="${mirror_host}/${url_path}${url_query}"
|
|
local local_checksum="${local_tarball}.${checksum_suffix}"
|
|
local remote_checksum="https://archive.apache.org/dist/${url_path}.${checksum_suffix}"
|
|
|
|
local curl_opts="--silent --show-error -L"
|
|
local wget_opts="--no-verbose"
|
|
|
|
if [ ! -f "$binary" ]; then
|
|
# check if we already have the tarball
|
|
# check if we have curl installed
|
|
# download application
|
|
if [ ! -f "${local_tarball}" -a "$(command -v curl)" ]; then
|
|
echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2
|
|
curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
|
|
if [ ! -z "${checksum_suffix}" ]; then
|
|
echo "exec: curl ${curl_opts} ${remote_checksum}" 1>&2
|
|
curl ${curl_opts} "${remote_checksum}" > "${local_checksum}"
|
|
fi
|
|
fi
|
|
# if the file still doesn't exist, lets try `wget` and cross our fingers
|
|
if [ ! -f "${local_tarball}" -a "$(command -v wget)" ]; then
|
|
echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2
|
|
wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
|
|
if [ ! -z "${checksum_suffix}" ]; then
|
|
echo "exec: wget ${wget_opts} ${remote_checksum}" 1>&2
|
|
wget ${wget_opts} -O "${local_checksum}" "${remote_checksum}"
|
|
fi
|
|
fi
|
|
# if both were unsuccessful, exit
|
|
if [ ! -f "${local_tarball}" ]; then
|
|
echo -n "ERROR: Cannot download ${remote_tarball} with cURL or wget; please install manually and try again."
|
|
exit 2
|
|
fi
|
|
# Checksum may not have been specified; don't check if doesn't exist
|
|
if [ -f "${local_checksum}" ]; then
|
|
echo " ${local_tarball}" >> ${local_checksum} # two spaces + file are important!
|
|
# Assuming SHA512 here for now
|
|
echo "Veryfing checksum from ${local_checksum}" 1>&2
|
|
if ! shasum -a 512 -q -c "${local_checksum}" ; then
|
|
echo "Bad checksum from ${remote_checksum}"
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
cd "${_DIR}" && tar -xzf "${local_tarball}"
|
|
rm -rf "${local_tarball}"
|
|
rm -f "${local_checksum}"
|
|
fi
|
|
}
|
|
|
|
# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
|
|
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
|
|
|
|
# Determine the Maven version from the root pom.xml file and
|
|
# install maven under the build/ folder if needed.
|
|
install_mvn() {
|
|
local MVN_VERSION=`grep "<maven.version>" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
|
|
MVN_BIN="$(command -v mvn)"
|
|
if [ "$MVN_BIN" ]; then
|
|
local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
|
|
fi
|
|
if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
|
|
local MVN_TARBALL="apache-maven-${MVN_VERSION}-bin.tar.gz"
|
|
local FILE_PATH="maven/maven-3/${MVN_VERSION}/binaries/${MVN_TARBALL}"
|
|
local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua'}
|
|
local MIRROR_URL_QUERY="?action=download"
|
|
|
|
if [ $(command -v curl) ]; then
|
|
if ! curl -L --output /dev/null --silent --head --fail "${APACHE_MIRROR}/${FILE_PATH}${MIRROR_URL_QUERY}" ; then
|
|
# Fall back to archive.apache.org for older Maven
|
|
echo "Falling back to archive.apache.org to download Maven"
|
|
APACHE_MIRROR="https://archive.apache.org/dist"
|
|
MIRROR_URL_QUERY=""
|
|
fi
|
|
fi
|
|
|
|
install_app \
|
|
"${APACHE_MIRROR}" \
|
|
"${FILE_PATH}" \
|
|
"${MIRROR_URL_QUERY}" \
|
|
"sha512" \
|
|
"${MVN_TARBALL}" \
|
|
"apache-maven-${MVN_VERSION}/bin/mvn"
|
|
|
|
MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
|
|
fi
|
|
}
|
|
|
|
# Determine the Scala version from the root pom.xml file, set the Scala URL,
|
|
# and, with that, download the specific version of Scala necessary under
|
|
# the build/ folder
|
|
install_scala() {
|
|
# determine the Scala version used in Spark
|
|
local scala_binary_version=`grep "scala.binary.version" "${_DIR}/../pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'`
|
|
local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | grep ${scala_binary_version} | head -n1 | awk -F '[<>]' '{print $3}'`
|
|
local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
|
|
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
|
|
local SCALA_TARBALL="scala-${scala_version}.tgz"
|
|
|
|
install_app \
|
|
"${TYPESAFE_MIRROR}" \
|
|
"scala/${scala_version}/${SCALA_TARBALL}" \
|
|
"" \
|
|
"" \
|
|
${SCALA_TARBALL} \
|
|
"scala-${scala_version}/bin/scala"
|
|
|
|
SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar"
|
|
SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar"
|
|
}
|
|
|
|
install_scala
|
|
install_mvn
|
|
|
|
# Reset the current working directory
|
|
cd "${_CALLING_DIR}"
|
|
|
|
# Set any `mvn` options if not already present
|
|
export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
|
|
|
|
echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
|
|
|
|
# call the `mvn` command as usual
|
|
# SPARK-25854
|
|
"${MVN_BIN}" "$@"
|
|
MVN_RETCODE=$?
|
|
|
|
exit $MVN_RETCODE
|