ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-32231][R][INFRA] Use Hadoop 3.2 winutils in AppVeyor build

Browse source 
### What changes were proposed in this pull request?

This PR proposes to use Hadoop 3 winutils to make AppVeyor builds pass. Currently it's being failed as below
https://ci.appveyor.com/project/ApacheSoftwareFoundation/spark/builds/33976604

### Why are the changes needed?

To recover the build in AppVeyor.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

AppVeyor build will test it out.

Closes #29042 from HyukjinKwon/SPARK-32231.

Lead-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Co-authored-by: Hyukjin Kwon <gurwls223@apache.org>
Co-authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
Dongjoon Hyun 2020-07-09 17:18:39 +09:00 committed by HyukjinKwon
parent 526cb2d1ba
commit c5bd0730a2
1 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
dev/appveyor-install-dependencies.ps1
View file

@ -67,8 +67,8 @@ Function InstallRtools {
Else {
$gccPath = $env:GCC_PATH
}
$env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\MinGW$(WIN)\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH
$env:BINPREF=$RtoolsDrive + '/Rtools40/mingw$(WIN)/bin/'
$env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\mingw64\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH
$env:BINPREF=$RtoolsDrive + '/Rtools40/mingw64/bin/'
}
# create tools directory outside of Spark directory
@ -95,22 +95,22 @@ $env:MAVEN_OPTS = "-Xmx2g -XX:ReservedCodeCacheSize=1g"
Pop-Location
# ========================== Hadoop bin package
# This must match the version at https://github.com/steveloughran/winutils/tree/master/hadoop-2.7.1
$hadoopVer = "2.7.1"
# This must match the version at https://github.com/cdarlint/winutils/tree/master/hadoop-3.2.0
$hadoopVer = "3.2.0"
$hadoopPath = "$tools\hadoop"
if (!(Test-Path $hadoopPath)) {
New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null
}
Push-Location $hadoopPath
Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip"
Start-FileDownload "https://codeload.github.com/cdarlint/winutils/zip/master" "winutils-master.zip"
# extract
Invoke-Expression "7z.exe x winutils-master.zip"
# add hadoop bin to environment variables
$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer"
$env:Path += ";$env:HADOOP_HOME\bin"
$env:HADOOP_HOME = "$hadoopPath\winutils-master\hadoop-$hadoopVer"
$env:PATH = "$env:HADOOP_HOME\bin;" + $env:PATH
Pop-Location