[SPARK-25335][BUILD] Skip Zinc downloading if it's installed in the system
## What changes were proposed in this pull request? Zinc is 23.5MB (tgz). ``` $ curl -LO https://downloads.lightbend.com/zinc/0.3.15/zinc-0.3.15.tgz % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 23.5M 100 23.5M 0 0 35.4M 0 --:--:-- --:--:-- --:--:-- 35.3M ``` Currently, Spark downloads Zinc once. However, it occurs too many times in build systems. This PR aims to skip Zinc downloading when the system already has it. ``` $ build/mvn clean exec: curl --progress-bar -L https://downloads.lightbend.com/zinc/0.3.15/zinc-0.3.15.tgz ######################################################################## 100.0% ``` This will reduce many resources(CPU/Networks/DISK) at least in Mac and Docker-based build system. ## How was this patch tested? Pass the Jenkins. Closes #22333 from dongjoon-hyun/SPARK-25335. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
parent
71bd796517
commit
458468ad51
29
build/mvn
29
build/mvn
|
@ -67,6 +67,9 @@ install_app() {
|
|||
fi
|
||||
}
|
||||
|
||||
# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
|
||||
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
|
||||
|
||||
# Determine the Maven version from the root pom.xml file and
|
||||
# install maven under the build/ folder if needed.
|
||||
install_mvn() {
|
||||
|
@ -75,8 +78,6 @@ install_mvn() {
|
|||
if [ "$MVN_BIN" ]; then
|
||||
local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
|
||||
fi
|
||||
# See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
|
||||
function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
|
||||
if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
|
||||
local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
|
||||
|
||||
|
@ -91,15 +92,23 @@ install_mvn() {
|
|||
|
||||
# Install zinc under the build/ folder
|
||||
install_zinc() {
|
||||
local zinc_path="zinc-0.3.15/bin/zinc"
|
||||
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
|
||||
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
|
||||
local ZINC_VERSION=0.3.15
|
||||
ZINC_BIN="$(command -v zinc)"
|
||||
if [ "$ZINC_BIN" ]; then
|
||||
local ZINC_DETECTED_VERSION="$(zinc -version | head -n1 | awk '{print $5}')"
|
||||
fi
|
||||
|
||||
install_app \
|
||||
"${TYPESAFE_MIRROR}/zinc/0.3.15" \
|
||||
"zinc-0.3.15.tgz" \
|
||||
"${zinc_path}"
|
||||
ZINC_BIN="${_DIR}/${zinc_path}"
|
||||
if [ $(version $ZINC_DETECTED_VERSION) -lt $(version $ZINC_VERSION) ]; then
|
||||
local zinc_path="zinc-${ZINC_VERSION}/bin/zinc"
|
||||
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
|
||||
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
|
||||
|
||||
install_app \
|
||||
"${TYPESAFE_MIRROR}/zinc/${ZINC_VERSION}" \
|
||||
"zinc-${ZINC_VERSION}.tgz" \
|
||||
"${zinc_path}"
|
||||
ZINC_BIN="${_DIR}/${zinc_path}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Determine the Scala version from the root pom.xml file, set the Scala URL,
|
||||
|
|
Loading…
Reference in a new issue