2018-06-22 13:38:34 -04:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
#
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
|
|
# this work for additional information regarding copyright ownership.
|
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
# (the "License"); you may not use this file except in compliance with
|
|
|
|
# the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
#
|
|
|
|
|
|
|
|
#
|
|
|
|
# Creates a Spark release candidate. The script will update versions, tag the branch,
|
|
|
|
# build Spark binary packages and documentation, and upload maven artifacts to a staging
|
|
|
|
# repository. There is also a dry run mode where only local builds are performed, and
|
|
|
|
# nothing is uploaded to the ASF repos.
|
|
|
|
#
|
|
|
|
# Run with "-h" for options.
|
|
|
|
#
|
|
|
|
|
|
|
|
set -e
|
|
|
|
SELF=$(cd $(dirname $0) && pwd)
|
|
|
|
. "$SELF/release-util.sh"
|
|
|
|
|
|
|
|
function usage {
|
|
|
|
local NAME=$(basename $0)
|
|
|
|
cat <<EOF
|
|
|
|
Usage: $NAME [options]
|
|
|
|
|
|
|
|
This script runs the release scripts inside a docker image. The image is hardcoded to be called
|
|
|
|
"spark-rm" and will be re-generated (as needed) on every invocation of this script.
|
|
|
|
|
|
|
|
Options are:
|
|
|
|
|
|
|
|
-d [path] : required: working directory (output will be written to an "output" directory in
|
|
|
|
the working directory).
|
|
|
|
-n : dry run mode. Performs checks and local builds, but do not upload anything.
|
|
|
|
-t [tag] : tag for the spark-rm docker image to use for building (default: "latest").
|
|
|
|
-j [path] : path to local JDK installation to use for building. By default the script will
|
|
|
|
use openjdk8 installed in the docker image.
|
|
|
|
-s [step] : runs a single step of the process; valid steps are: tag, build, docs, publish
|
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
WORKDIR=
|
|
|
|
IMGTAG=latest
|
|
|
|
JAVA=
|
|
|
|
RELEASE_STEP=
|
2020-04-16 15:54:10 -04:00
|
|
|
while getopts ":d:hj:ns:t:" opt; do
|
2018-06-22 13:38:34 -04:00
|
|
|
case $opt in
|
|
|
|
d) WORKDIR="$OPTARG" ;;
|
|
|
|
n) DRY_RUN=1 ;;
|
|
|
|
t) IMGTAG="$OPTARG" ;;
|
|
|
|
j) JAVA="$OPTARG" ;;
|
|
|
|
s) RELEASE_STEP="$OPTARG" ;;
|
|
|
|
h) usage ;;
|
2020-04-16 15:54:10 -04:00
|
|
|
\?) error "Invalid option. Run with -h for help." ;;
|
2018-06-22 13:38:34 -04:00
|
|
|
esac
|
|
|
|
done
|
|
|
|
|
|
|
|
if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
|
|
|
|
error "Work directory (-d) must be defined and exist. Run with -h for help."
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -d "$WORKDIR/output" ]; then
|
|
|
|
read -p "Output directory already exists. Overwrite and continue? [y/n] " ANSWER
|
|
|
|
if [ "$ANSWER" != "y" ]; then
|
|
|
|
error "Exiting."
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
cd "$WORKDIR"
|
|
|
|
rm -rf "$WORKDIR/output"
|
|
|
|
mkdir "$WORKDIR/output"
|
|
|
|
|
|
|
|
get_release_info
|
|
|
|
|
|
|
|
# Place all RM scripts and necessary data in a local directory that must be defined in the command
|
|
|
|
# line. This directory is mounted into the image.
|
|
|
|
for f in "$SELF"/*; do
|
|
|
|
if [ -f "$f" ]; then
|
|
|
|
cp "$f" "$WORKDIR"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
[SPARK-34433][DOCS] Lock Jekyll version by Gemfile and Bundler
### What changes were proposed in this pull request?
Improving the documentation and release process by pinning Jekyll version by Gemfile and Bundler.
Some files and their responsibilities within this PR:
- `docs/.bundle/config` is used to specify a directory "docs/.local_ruby_bundle" which will be used as destination to install the ruby packages into instead of the global one which requires root access
- `docs/Gemfile` is specifying the required Jekyll version and other top level gem versions
- `docs/Gemfile.lock` is generated by the "bundle install". This file contains the exact resolved versions of all the gems including the top level gems and all the direct and transitive dependencies of those gems. When this file is generated it contains a platform related section "PLATFORMS" (in my case after the generation it was "universal-darwin-19"). Still this file must be under version control as when the version of a gem does not fit to the one specified in `Gemfile` an error comes (i.e. if the `Gemfile.lock` was generated for Jekyll 4.1.0 and its version is updated in the `Gemfile` to 4.2.0 then it triggers the error: "The bundle currently has jekyll locked at 4.1.0."). This is solution is also suggested officially in [its documentation](https://bundler.io/rationale.html#checking-your-code-into-version-control). To get rid of the specific platform (like "universal-darwin-19") first we have to add "ruby" as platform [which means this should work on every platform where Ruby runs](https://guides.rubygems.org/what-is-a-gem/)) by running "bundle lock --add-platform ruby" then the specific platform can be removed by "bundle lock --remove-platform universal-darwin-19".
After this the correct process to update Jekyll version is the following:
1. update the version in `Gemfile`
2. run "bundle update" which updates the `Gemfile.lock`
3. commit both files
This process for version update is tested for details please check the testing section.
### Why are the changes needed?
Using different Jekyll versions can generate different output documents.
This PR standardize the process.
### Does this PR introduce _any_ user-facing change?
No, assuming the release was done via docker by using `do-release-docker.sh`.
In that case there should be no difference at all as the same Jekyll version is specified in the Gemfile.
### How was this patch tested?
#### Testing document generation
Doc generation step was triggered via the docker release:
```
$ ./do-release-docker.sh -d ~/working -n -s docs
...
========================
= Building documentation...
Command: /opt/spark-rm/release-build.sh docs
Log file: docs.log
Skipping publish step.
```
The docs.log contains the followings:
```
Building Spark docs
Fetching gem metadata from https://rubygems.org/.........
Using bundler 2.2.9
Fetching rb-fsevent 0.10.4
Fetching forwardable-extended 2.6.0
Fetching public_suffix 4.0.6
Fetching colorator 1.1.0
Fetching eventmachine 1.2.7
Fetching http_parser.rb 0.6.0
Fetching ffi 1.14.2
Fetching concurrent-ruby 1.1.8
Installing colorator 1.1.0
Installing forwardable-extended 2.6.0
Installing rb-fsevent 0.10.4
Installing public_suffix 4.0.6
Installing http_parser.rb 0.6.0 with native extensions
Installing eventmachine 1.2.7 with native extensions
Installing concurrent-ruby 1.1.8
Fetching rexml 3.2.4
Fetching liquid 4.0.3
Installing ffi 1.14.2 with native extensions
Installing rexml 3.2.4
Installing liquid 4.0.3
Fetching mercenary 0.4.0
Installing mercenary 0.4.0
Fetching rouge 3.26.0
Installing rouge 3.26.0
Fetching safe_yaml 1.0.5
Installing safe_yaml 1.0.5
Fetching unicode-display_width 1.7.0
Installing unicode-display_width 1.7.0
Fetching webrick 1.7.0
Installing webrick 1.7.0
Fetching pathutil 0.16.2
Fetching kramdown 2.3.0
Fetching terminal-table 2.0.0
Fetching addressable 2.7.0
Fetching i18n 1.8.9
Installing terminal-table 2.0.0
Installing pathutil 0.16.2
Installing i18n 1.8.9
Installing addressable 2.7.0
Installing kramdown 2.3.0
Fetching kramdown-parser-gfm 1.1.0
Installing kramdown-parser-gfm 1.1.0
Fetching rb-inotify 0.10.1
Fetching sassc 2.4.0
Fetching em-websocket 0.5.2
Installing rb-inotify 0.10.1
Installing em-websocket 0.5.2
Installing sassc 2.4.0 with native extensions
Fetching listen 3.4.1
Installing listen 3.4.1
Fetching jekyll-watch 2.2.1
Installing jekyll-watch 2.2.1
Fetching jekyll-sass-converter 2.1.0
Installing jekyll-sass-converter 2.1.0
Fetching jekyll 4.2.0
Installing jekyll 4.2.0
Fetching jekyll-redirect-from 0.16.0
Installing jekyll-redirect-from 0.16.0
Bundle complete! 4 Gemfile dependencies, 30 gems now installed.
Bundled gems are installed into `./.local_ruby_bundle`
```
#### Testing Jekyll (or other gem) update
First locally I reverted Jekyll to 4.1.0:
```
$ rm Gemfile.lock
$ rm -rf .local_ruby_bundle
# edited Gemfile to use version 4.1.0
$ cat Gemfile
source "https://rubygems.org"
gem "jekyll", "4.1.0"
gem "rouge", "3.26.0"
gem "jekyll-redirect-from", "0.16.0"
gem "webrick", "1.7"
$ bundle install
...
```
Testing Jekyll version before the update:
```
$ bundle exec jekyll --version
jekyll 4.1.0
```
Imitating Jekyll update coming from git by reverting my local changes:
```
$ git checkout Gemfile
Updated 1 path from the index
$ cat Gemfile
source "https://rubygems.org"
gem "jekyll", "4.2.0"
gem "rouge", "3.26.0"
gem "jekyll-redirect-from", "0.16.0"
gem "webrick", "1.7"
$ git checkout Gemfile.lock
Updated 1 path from the index
```
Run the install:
```
$ bundle install
...
```
Checking the updated Jekyll version:
```
$ bundle exec jekyll --version
jekyll 4.2.0
```
Closes #31559 from attilapiros/pin-jekyll-version.
Lead-authored-by: “attilapiros” <piros.attila.zsolt@gmail.com>
Co-authored-by: Hyukjin Kwon <gurwls223@gmail.com>
Co-authored-by: Attila Zsolt Piros <2017933+attilapiros@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
2021-02-17 22:17:57 -05:00
|
|
|
# Add the fallback version of Gemfile, Gemfile.lock and .bundle/config to the local directory.
|
|
|
|
cp "$SELF/../../docs/Gemfile" "$WORKDIR"
|
|
|
|
cp "$SELF/../../docs/Gemfile.lock" "$WORKDIR"
|
|
|
|
cp -r "$SELF/../../docs/.bundle" "$WORKDIR"
|
|
|
|
|
2018-06-22 13:38:34 -04:00
|
|
|
GPG_KEY_FILE="$WORKDIR/gpg.key"
|
|
|
|
fcreate_secure "$GPG_KEY_FILE"
|
2020-04-01 03:43:32 -04:00
|
|
|
$GPG --export-secret-key --armor --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" "$GPG_KEY" > "$GPG_KEY_FILE"
|
2018-06-22 13:38:34 -04:00
|
|
|
|
|
|
|
run_silent "Building spark-rm image with tag $IMGTAG..." "docker-build.log" \
|
2020-03-31 03:11:59 -04:00
|
|
|
docker build -t "spark-rm:$IMGTAG" --build-arg UID=$UID "$SELF/spark-rm"
|
2018-06-22 13:38:34 -04:00
|
|
|
|
|
|
|
# Write the release information to a file with environment variables to be used when running the
|
|
|
|
# image.
|
|
|
|
ENVFILE="$WORKDIR/env.list"
|
|
|
|
fcreate_secure "$ENVFILE"
|
|
|
|
|
|
|
|
function cleanup {
|
|
|
|
rm -f "$ENVFILE"
|
|
|
|
rm -f "$GPG_KEY_FILE"
|
|
|
|
}
|
|
|
|
|
|
|
|
trap cleanup EXIT
|
|
|
|
|
|
|
|
cat > $ENVFILE <<EOF
|
|
|
|
DRY_RUN=$DRY_RUN
|
|
|
|
SKIP_TAG=$SKIP_TAG
|
|
|
|
RUNNING_IN_DOCKER=1
|
|
|
|
GIT_BRANCH=$GIT_BRANCH
|
|
|
|
NEXT_VERSION=$NEXT_VERSION
|
|
|
|
RELEASE_VERSION=$RELEASE_VERSION
|
|
|
|
RELEASE_TAG=$RELEASE_TAG
|
|
|
|
GIT_REF=$GIT_REF
|
|
|
|
SPARK_PACKAGE_VERSION=$SPARK_PACKAGE_VERSION
|
|
|
|
ASF_USERNAME=$ASF_USERNAME
|
|
|
|
GIT_NAME=$GIT_NAME
|
|
|
|
GIT_EMAIL=$GIT_EMAIL
|
|
|
|
GPG_KEY=$GPG_KEY
|
|
|
|
ASF_PASSWORD=$ASF_PASSWORD
|
|
|
|
GPG_PASSPHRASE=$GPG_PASSPHRASE
|
|
|
|
RELEASE_STEP=$RELEASE_STEP
|
[SPARK-28906][BUILD] Fix incorrect information in bin/spark-submit --version
### What changes were proposed in this pull request?
This PR allows `bin/spark-submit --version` to show the correct information while the previous versions, which were created by `dev/create-release/do-release-docker.sh`, show incorrect information.
There are two root causes to show incorrect information:
1. Did not pass `USER` environment variable to the docker container
1. Did not keep `.git` directory in the work directory
### Why are the changes needed?
The information is missing while the previous versions show the correct information.
### Does this PR introduce any user-facing change?
Yes, the following is the console output in branch-2.3
```
$ bin/spark-submit --version
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.3.4
/_/
Using Scala version 2.11.8, OpenJDK 64-Bit Server VM, 1.8.0_212
Branch HEAD
Compiled by user ishizaki on 2019-09-02T02:18:10Z
Revision 8c6f8150f3c6298ff4e1c7e06028f12d7eaf0210
Url https://gitbox.apache.org/repos/asf/spark.git
Type --help for more information.
```
Without this PR, the console output is as follows
```
$ spark-submit --version
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.3.4
/_/
Using Scala version 2.11.8, OpenJDK 64-Bit Server VM, 1.8.0_212
Branch
Compiled by user on 2019-08-26T08:29:39Z
Revision
Url
Type --help for more information.
```
### How was this patch tested?
After building the package, I manually executed `bin/spark-submit --version`
Closes #25655 from kiszk/SPARK-28906.
Authored-by: Kazuaki Ishizaki <ishizaki@jp.ibm.com>
Signed-off-by: Sean Owen <sean.owen@databricks.com>
2019-09-11 09:12:44 -04:00
|
|
|
USER=$USER
|
2018-06-22 13:38:34 -04:00
|
|
|
EOF
|
|
|
|
|
|
|
|
JAVA_VOL=
|
|
|
|
if [ -n "$JAVA" ]; then
|
|
|
|
echo "JAVA_HOME=/opt/spark-java" >> $ENVFILE
|
|
|
|
JAVA_VOL="--volume $JAVA:/opt/spark-java"
|
|
|
|
fi
|
|
|
|
|
|
|
|
echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
|
|
|
|
docker run -ti \
|
|
|
|
--env-file "$ENVFILE" \
|
|
|
|
--volume "$WORKDIR:/opt/spark-rm" \
|
|
|
|
$JAVA_VOL \
|
|
|
|
"spark-rm:$IMGTAG"
|