2019-08-13 18:55:02 -04:00
|
|
|
name: master
|
|
|
|
|
|
|
|
on:
|
|
|
|
push:
|
|
|
|
branches:
|
|
|
|
- master
|
2019-09-13 17:50:06 -04:00
|
|
|
pull_request:
|
|
|
|
branches:
|
|
|
|
- master
|
2019-08-13 18:55:02 -04:00
|
|
|
|
|
|
|
jobs:
|
|
|
|
build:
|
|
|
|
|
|
|
|
runs-on: ubuntu-latest
|
2019-08-13 23:14:07 -04:00
|
|
|
strategy:
|
|
|
|
matrix:
|
|
|
|
java: [ '1.8', '11' ]
|
2019-09-17 19:53:21 -04:00
|
|
|
hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
|
2019-11-23 13:02:22 -05:00
|
|
|
hive: [ 'hive-1.2', 'hive-2.3' ]
|
2019-09-17 19:53:21 -04:00
|
|
|
exclude:
|
2019-11-23 13:02:22 -05:00
|
|
|
- java: '11'
|
|
|
|
hive: 'hive-1.2'
|
|
|
|
- hadoop: 'hadoop-3.2'
|
|
|
|
hive: 'hive-1.2'
|
|
|
|
name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
|
2019-08-13 18:55:02 -04:00
|
|
|
|
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@master
|
2019-11-10 14:02:54 -05:00
|
|
|
# We split caches because GitHub Action Cache has a 400MB-size limit.
|
2019-11-24 15:35:57 -05:00
|
|
|
- uses: actions/cache@v1
|
|
|
|
with:
|
|
|
|
path: build
|
|
|
|
key: build-${{ hashFiles('**/pom.xml') }}
|
|
|
|
restore-keys: |
|
|
|
|
build-
|
2019-11-10 14:02:54 -05:00
|
|
|
- uses: actions/cache@v1
|
|
|
|
with:
|
|
|
|
path: ~/.m2/repository/com
|
|
|
|
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
|
|
|
|
restore-keys: |
|
|
|
|
${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
|
|
|
|
- uses: actions/cache@v1
|
|
|
|
with:
|
|
|
|
path: ~/.m2/repository/org
|
|
|
|
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
|
|
|
|
restore-keys: |
|
|
|
|
${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
|
2019-11-21 01:43:57 -05:00
|
|
|
- uses: actions/cache@v1
|
|
|
|
with:
|
|
|
|
path: ~/.m2/repository/net
|
|
|
|
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
|
|
|
|
restore-keys: |
|
|
|
|
${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
|
|
|
|
- uses: actions/cache@v1
|
|
|
|
with:
|
|
|
|
path: ~/.m2/repository/io
|
|
|
|
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
|
|
|
|
restore-keys: |
|
|
|
|
${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
|
2019-08-13 23:14:07 -04:00
|
|
|
- name: Set up JDK ${{ matrix.java }}
|
2019-08-13 18:55:02 -04:00
|
|
|
uses: actions/setup-java@v1
|
|
|
|
with:
|
2019-09-20 11:54:34 -04:00
|
|
|
java-version: ${{ matrix.java }}
|
2019-08-13 18:55:02 -04:00
|
|
|
- name: Build with Maven
|
|
|
|
run: |
|
2019-09-19 03:24:15 -04:00
|
|
|
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
|
2019-08-15 18:22:44 -04:00
|
|
|
export MAVEN_CLI_OPTS="--no-transfer-progress"
|
2019-12-07 15:58:00 -05:00
|
|
|
mkdir -p ~/.m2
|
2019-11-23 13:02:22 -05:00
|
|
|
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
|
2019-11-10 14:02:54 -05:00
|
|
|
rm -rf ~/.m2/repository/org/apache/spark
|
2019-09-21 11:13:00 -04:00
|
|
|
|
|
|
|
|
|
|
|
lint:
|
|
|
|
runs-on: ubuntu-latest
|
2019-11-18 00:01:01 -05:00
|
|
|
name: Linters (Java/Scala/Python), licenses, dependencies
|
2019-09-21 11:13:00 -04:00
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@master
|
|
|
|
- uses: actions/setup-java@v1
|
|
|
|
with:
|
|
|
|
java-version: '11'
|
|
|
|
- uses: actions/setup-python@v1
|
|
|
|
with:
|
|
|
|
python-version: '3.x'
|
|
|
|
architecture: 'x64'
|
|
|
|
- name: Scala
|
|
|
|
run: ./dev/lint-scala
|
|
|
|
- name: Java
|
|
|
|
run: ./dev/lint-java
|
|
|
|
- name: Python
|
|
|
|
run: |
|
|
|
|
pip install flake8 sphinx numpy
|
|
|
|
./dev/lint-python
|
|
|
|
- name: License
|
|
|
|
run: ./dev/check-license
|
|
|
|
- name: Dependencies
|
|
|
|
run: ./dev/test-dependencies.sh
|
2019-11-18 00:01:01 -05:00
|
|
|
|
|
|
|
lintr:
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
name: Linter (R)
|
|
|
|
steps:
|
|
|
|
- uses: actions/checkout@master
|
|
|
|
- uses: actions/setup-java@v1
|
|
|
|
with:
|
|
|
|
java-version: '11'
|
|
|
|
- name: install R
|
|
|
|
run: |
|
|
|
|
echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
|
2019-11-19 18:56:50 -05:00
|
|
|
curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
|
2019-11-18 00:01:01 -05:00
|
|
|
sudo apt-get update
|
|
|
|
sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
|
|
|
|
- name: install R packages
|
|
|
|
run: |
|
|
|
|
sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
|
|
|
|
sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
|
|
|
|
- name: package and install SparkR
|
|
|
|
run: ./R/install-dev.sh
|
|
|
|
- name: lint-r
|
|
|
|
run: ./dev/lint-r
|