SPARK-4466: Provide support for publishing Scala 2.11 artifacts to Maven
The maven release plug-in does not have support for publishing two separate sets of artifacts for a single release. Because of the way that Scala 2.11 support in Spark works, we have to write some customized code to do this. The good news is that the Maven release API is just a thin wrapper on doing git commits and pushing artifacts to the HTTP API of Apache's Sonatype server and this might overall make our deployment easier to understand. This was already used for the 1.2 snapshot, so I think it is working well. One other nice thing is this could be pretty easily extended to publish nightly snapshots. Author: Patrick Wendell <pwendell@gmail.com> Closes #3332 from pwendell/releases and squashes the following commits: 2fedaed [Patrick Wendell] Automate the opening and closing of Sonatype repos e2a24bb [Patrick Wendell] Fixing issue where we overrode non-spark version numbers 9df3a50 [Patrick Wendell] Adding TODO 1cc1749 [Patrick Wendell] Don't build the thriftserver for 2.11 933201a [Patrick Wendell] Make tagging of release commit eager d0388a6 [Patrick Wendell] Support Scala 2.11 build 4f4dc62 [Patrick Wendell] Change to 2.11 should not be included when committing new patch bf742e1 [Patrick Wendell] Minor fixes ffa1df2 [Patrick Wendell] Adding a Scala 2.11 package to test it 9ac4381 [Patrick Wendell] Addressing TODO b3105ff [Patrick Wendell] Removing commented out code d906803 [Patrick Wendell] Small fix 3f4d985 [Patrick Wendell] More work fcd54c2 [Patrick Wendell] Consolidating use of keys df2af30 [Patrick Wendell] Changes to release stuff
This commit is contained in:
Patrick Wendell
parent
36b0956a3e
commit
c6e0c2ab1c
|
|
@ -22,4 +22,4 @@ package org.apache.spark.api.java
|
|||
* these interfaces to pass functions to various Java API methods for Spark. Please visit Spark's
|
||||
* Java programming guide for more details.
|
||||
*/
|
||||
package object function
|
||||
package object function
|
||||
|
|
|
|||
|
|
@ -28,13 +28,19 @@
|
|||
# - Send output to stderr and have useful logging in stdout
|
||||
|
||||
# Note: The following variables must be set before use!
|
||||
GIT_USERNAME=${GIT_USERNAME:-pwendell}
|
||||
GIT_PASSWORD=${GIT_PASSWORD:-XXX}
|
||||
ASF_USERNAME=${ASF_USERNAME:-pwendell}
|
||||
ASF_PASSWORD=${ASF_PASSWORD:-XXX}
|
||||
GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
|
||||
GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
|
||||
RELEASE_VERSION=${RELEASE_VERSION:-1.0.0}
|
||||
RELEASE_VERSION=${RELEASE_VERSION:-1.2.0}
|
||||
NEXT_VERSION=${NEXT_VERSION:-1.2.1}
|
||||
RC_NAME=${RC_NAME:-rc2}
|
||||
USER_NAME=${USER_NAME:-pwendell}
|
||||
|
||||
M2_REPO=~/.m2/repository
|
||||
SPARK_REPO=$M2_REPO/org/apache/spark
|
||||
NEXUS_ROOT=https://repository.apache.org/service/local/staging
|
||||
NEXUS_UPLOAD=$NEXUS_ROOT/deploy/maven2
|
||||
NEXUS_PROFILE=d63f592e7eac0 # Profile for Spark staging uploads
|
||||
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
echo "Error: JAVA_HOME is not set, cannot proceed."
|
||||
|
|
@ -47,31 +53,90 @@ set -e
|
|||
GIT_TAG=v$RELEASE_VERSION-$RC_NAME
|
||||
|
||||
if [[ ! "$@" =~ --package-only ]]; then
|
||||
echo "Creating and publishing release"
|
||||
echo "Creating release commit and publishing to Apache repository"
|
||||
# Artifact publishing
|
||||
git clone https://git-wip-us.apache.org/repos/asf/spark.git -b $GIT_BRANCH
|
||||
cd spark
|
||||
git clone https://$ASF_USERNAME:$ASF_PASSWORD@git-wip-us.apache.org/repos/asf/spark.git \
|
||||
-b $GIT_BRANCH
|
||||
pushd spark
|
||||
export MAVEN_OPTS="-Xmx3g -XX:MaxPermSize=1g -XX:ReservedCodeCacheSize=1g"
|
||||
|
||||
mvn -Pyarn release:clean
|
||||
# Create release commits and push them to github
|
||||
# NOTE: This is done "eagerly" i.e. we don't check if we can succesfully build
|
||||
# or before we coin the release commit. This helps avoid races where
|
||||
# other people add commits to this branch while we are in the middle of building.
|
||||
old=" <version>${RELEASE_VERSION}-SNAPSHOT<\/version>"
|
||||
new=" <version>${RELEASE_VERSION}<\/version>"
|
||||
find . -name pom.xml -o -name package.scala | grep -v dev | xargs -I {} sed -i \
|
||||
-e "s/$old/$new/" {}
|
||||
git commit -a -m "Preparing Spark release $GIT_TAG"
|
||||
echo "Creating tag $GIT_TAG at the head of $GIT_BRANCH"
|
||||
git tag $GIT_TAG
|
||||
|
||||
mvn -DskipTests \
|
||||
-Darguments="-DskipTests=true -Dmaven.javadoc.skip=true -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -Dgpg.passphrase=${GPG_PASSPHRASE}" \
|
||||
-Dusername=$GIT_USERNAME -Dpassword=$GIT_PASSWORD \
|
||||
-Dmaven.javadoc.skip=true \
|
||||
-Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
|
||||
-Dtag=$GIT_TAG -DautoVersionSubmodules=true \
|
||||
old=" <version>${RELEASE_VERSION}<\/version>"
|
||||
new=" <version>${NEXT_VERSION}-SNAPSHOT<\/version>"
|
||||
find . -name pom.xml -o -name package.scala | grep -v dev | xargs -I {} sed -i \
|
||||
-e "s/$old/$new/" {}
|
||||
git commit -a -m "Preparing development version ${NEXT_VERSION}-SNAPSHOT"
|
||||
git push origin $GIT_TAG
|
||||
git push origin HEAD:$GIT_BRANCH
|
||||
git checkout -f $GIT_TAG
|
||||
|
||||
# Using Nexus API documented here:
|
||||
# https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API
|
||||
echo "Creating Nexus staging repository"
|
||||
repo_request="<promoteRequest><data><description>Apache Spark $GIT_TAG</description></data></promoteRequest>"
|
||||
out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \
|
||||
-H "Content-Type:application/xml" -v \
|
||||
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start)
|
||||
staged_repo_id=$(echo $out | sed -e "s/.*\(orgapachespark-[0-9]\{4\}\).*/\1/")
|
||||
echo "Created Nexus staging repository: $staged_repo_id"
|
||||
|
||||
rm -rf $SPARK_REPO
|
||||
|
||||
mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
|
||||
-Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
|
||||
--batch-mode release:prepare
|
||||
clean install
|
||||
|
||||
mvn -DskipTests \
|
||||
-Darguments="-DskipTests=true -Dmaven.javadoc.skip=true -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -Dgpg.passphrase=${GPG_PASSPHRASE}" \
|
||||
-Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
|
||||
-Dmaven.javadoc.skip=true \
|
||||
-Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
|
||||
release:perform
|
||||
./dev/change-version-to-2.11.sh
|
||||
|
||||
mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
|
||||
-Dscala-2.11 -Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
|
||||
clean install
|
||||
|
||||
cd ..
|
||||
./dev/change-version-to-2.10.sh
|
||||
|
||||
pushd $SPARK_REPO
|
||||
|
||||
# Remove any extra files generated during install
|
||||
find . -type f |grep -v \.jar |grep -v \.pom | xargs rm
|
||||
|
||||
echo "Creating hash and signature files"
|
||||
for file in $(find . -type f)
|
||||
do
|
||||
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --output $file.asc --detach-sig --armour $file;
|
||||
gpg --print-md MD5 $file > $file.md5;
|
||||
gpg --print-md SHA1 $file > $file.sha1
|
||||
done
|
||||
|
||||
echo "Uplading files to $NEXUS_UPLOAD"
|
||||
for file in $(find . -type f)
|
||||
do
|
||||
# strip leading ./
|
||||
file_short=$(echo $file | sed -e "s/\.\///")
|
||||
dest_url="$NEXUS_UPLOAD/org/apache/spark/$file_short"
|
||||
echo " Uploading $file_short"
|
||||
curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $file_short $dest_url
|
||||
done
|
||||
|
||||
echo "Closing nexus staging repository"
|
||||
repo_request="<promoteRequest><data><stagedRepositoryId>$staged_repo_id</stagedRepositoryId><description>Apache Spark $GIT_TAG</description></data></promoteRequest>"
|
||||
out=$(curl -X POST -d "$repo_request" -u $ASF_USERNAME:$ASF_PASSWORD \
|
||||
-H "Content-Type:application/xml" -v \
|
||||
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish)
|
||||
echo "Closed Nexus staging repository: $staged_repo_id"
|
||||
|
||||
popd
|
||||
popd
|
||||
rm -rf spark
|
||||
fi
|
||||
|
||||
|
|
@ -102,6 +167,12 @@ make_binary_release() {
|
|||
cp -r spark spark-$RELEASE_VERSION-bin-$NAME
|
||||
|
||||
cd spark-$RELEASE_VERSION-bin-$NAME
|
||||
|
||||
# TODO There should probably be a flag to make-distribution to allow 2.11 support
|
||||
if [[ $FLAGS == *scala-2.11* ]]; then
|
||||
./dev/change-version-to-2.11.sh
|
||||
fi
|
||||
|
||||
./make-distribution.sh --name $NAME --tgz $FLAGS 2>&1 | tee ../binary-release-$NAME.log
|
||||
cd ..
|
||||
cp spark-$RELEASE_VERSION-bin-$NAME/spark-$RELEASE_VERSION-bin-$NAME.tgz .
|
||||
|
|
@ -118,11 +189,12 @@ make_binary_release() {
|
|||
spark-$RELEASE_VERSION-bin-$NAME.tgz.sha
|
||||
}
|
||||
|
||||
|
||||
make_binary_release "hadoop1" "-Phive -Phive-thriftserver -Dhadoop.version=1.0.4" &
|
||||
make_binary_release "hadoop1-scala2.11" "-Phive -Dscala-2.11" &
|
||||
make_binary_release "cdh4" "-Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" &
|
||||
make_binary_release "hadoop2.3" "-Phadoop-2.3 -Phive -Phive-thriftserver -Pyarn" &
|
||||
make_binary_release "hadoop2.4" "-Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn" &
|
||||
make_binary_release "hadoop2.4-without-hive" "-Phadoop-2.4 -Pyarn" &
|
||||
make_binary_release "mapr3" "-Pmapr3 -Phive -Phive-thriftserver" &
|
||||
make_binary_release "mapr4" "-Pmapr4 -Pyarn -Phive -Phive-thriftserver" &
|
||||
wait
|
||||
|
|
@ -130,10 +202,10 @@ wait
|
|||
# Copy data
|
||||
echo "Copying release tarballs"
|
||||
rc_folder=spark-$RELEASE_VERSION-$RC_NAME
|
||||
ssh $USER_NAME@people.apache.org \
|
||||
mkdir /home/$USER_NAME/public_html/$rc_folder
|
||||
ssh $ASF_USERNAME@people.apache.org \
|
||||
mkdir /home/$ASF_USERNAME/public_html/$rc_folder
|
||||
scp spark-* \
|
||||
$USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$rc_folder/
|
||||
$ASF_USERNAME@people.apache.org:/home/$ASF_USERNAME/public_html/$rc_folder/
|
||||
|
||||
# Docs
|
||||
cd spark
|
||||
|
|
@ -143,12 +215,12 @@ cd docs
|
|||
JAVA_HOME=$JAVA_7_HOME PRODUCTION=1 jekyll build
|
||||
echo "Copying release documentation"
|
||||
rc_docs_folder=${rc_folder}-docs
|
||||
ssh $USER_NAME@people.apache.org \
|
||||
mkdir /home/$USER_NAME/public_html/$rc_docs_folder
|
||||
rsync -r _site/* $USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$rc_docs_folder
|
||||
ssh $ASF_USERNAME@people.apache.org \
|
||||
mkdir /home/$ASF_USERNAME/public_html/$rc_docs_folder
|
||||
rsync -r _site/* $ASF_USERNAME@people.apache.org:/home/$ASF_USERNAME/public_html/$rc_docs_folder
|
||||
|
||||
echo "Release $RELEASE_VERSION completed:"
|
||||
echo "Git tag:\t $GIT_TAG"
|
||||
echo "Release commit:\t $release_hash"
|
||||
echo "Binary location:\t http://people.apache.org/~$USER_NAME/$rc_folder"
|
||||
echo "Doc location:\t http://people.apache.org/~$USER_NAME/$rc_docs_folder"
|
||||
echo "Binary location:\t http://people.apache.org/~$ASF_USERNAME/$rc_folder"
|
||||
echo "Doc location:\t http://people.apache.org/~$ASF_USERNAME/$rc_docs_folder"
|
||||
|
|
|
|||
Loading…
Reference in a new issue