3bf43fb60d
### What changes were proposed in this pull request? This PR aims to increase the JVM CodeCacheSize from 0.5G to 1G. ### Why are the changes needed? After upgrading to `Scala 2.12.10`, the following is observed during building. ``` 2019-09-18T20:49:23.5030586Z OpenJDK 64-Bit Server VM warning: CodeCache is full. Compiler has been disabled. 2019-09-18T20:49:23.5032920Z OpenJDK 64-Bit Server VM warning: Try increasing the code cache size using -XX:ReservedCodeCacheSize= 2019-09-18T20:49:23.5034959Z CodeCache: size=524288Kb used=521399Kb max_used=521423Kb free=2888Kb 2019-09-18T20:49:23.5035472Z bounds [0x00007fa62c000000, 0x00007fa64c000000, 0x00007fa64c000000] 2019-09-18T20:49:23.5035781Z total_blobs=156549 nmethods=155863 adapters=592 2019-09-18T20:49:23.5036090Z compilation: disabled (not enough contiguous free space left) ``` ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Manually check the Jenkins or GitHub Action build log (which should not have the above). Closes #25836 from dongjoon-hyun/SPARK-CODE-CACHE-1G. Authored-by: Dongjoon Hyun <dhyun@apple.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
129 lines
3.8 KiB
PowerShell
129 lines
3.8 KiB
PowerShell
<#
|
|
Licensed to the Apache Software Foundation (ASF) under one
|
|
or more contributor license agreements. See the NOTICE file
|
|
distributed with this work for additional information
|
|
regarding copyright ownership. The ASF licenses this file
|
|
to you under the Apache License, Version 2.0 (the
|
|
"License"); you may not use this file except in compliance
|
|
with the License. You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing,
|
|
software distributed under the License is distributed on an
|
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
KIND, either express or implied. See the License for the
|
|
specific language governing permissions and limitations
|
|
under the License.
|
|
#>
|
|
|
|
$CRAN = "https://cloud.r-project.org"
|
|
|
|
Function InstallR {
|
|
if ( -not(Test-Path Env:\R_ARCH) ) {
|
|
$arch = "i386"
|
|
}
|
|
Else {
|
|
$arch = $env:R_ARCH
|
|
}
|
|
|
|
$urlPath = ""
|
|
$latestVer = $(ConvertFrom-JSON $(Invoke-WebRequest https://rversions.r-pkg.org/r-release-win).Content).version
|
|
If ($rVer -ne $latestVer) {
|
|
$urlPath = ("old/" + $rVer + "/")
|
|
}
|
|
|
|
$rurl = $CRAN + "/bin/windows/base/" + $urlPath + "R-" + $rVer + "-win.exe"
|
|
|
|
# Downloading R
|
|
Start-FileDownload $rurl "R-win.exe"
|
|
|
|
# Running R installer
|
|
Start-Process -FilePath .\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
|
|
|
|
$RDrive = "C:"
|
|
echo "R is now available on drive $RDrive"
|
|
|
|
$env:PATH = $RDrive + '\R\bin\' + $arch + ';' + 'C:\MinGW\msys\1.0\bin;' + $env:PATH
|
|
|
|
# Testing R installation
|
|
Rscript -e "sessionInfo()"
|
|
}
|
|
|
|
Function InstallRtools {
|
|
$rtoolsver = $rToolsVer.Split('.')[0..1] -Join ''
|
|
$rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe"
|
|
|
|
# Downloading Rtools
|
|
Start-FileDownload $rtoolsurl "Rtools-current.exe"
|
|
|
|
# Running Rtools installer
|
|
Start-Process -FilePath .\Rtools-current.exe -ArgumentList /VERYSILENT -NoNewWindow -Wait
|
|
|
|
$RtoolsDrive = "C:"
|
|
echo "Rtools is now available on drive $RtoolsDrive"
|
|
|
|
if ( -not(Test-Path Env:\GCC_PATH) ) {
|
|
$gccPath = "gcc-4.6.3"
|
|
}
|
|
Else {
|
|
$gccPath = $env:GCC_PATH
|
|
}
|
|
$env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH
|
|
$env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/'
|
|
}
|
|
|
|
# create tools directory outside of Spark directory
|
|
$up = (Get-Item -Path ".." -Verbose).FullName
|
|
$tools = "$up\tools"
|
|
if (!(Test-Path $tools)) {
|
|
New-Item -ItemType Directory -Force -Path $tools | Out-Null
|
|
}
|
|
|
|
# ========================== Maven
|
|
Push-Location $tools
|
|
|
|
$mavenVer = "3.6.2"
|
|
Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip"
|
|
|
|
# extract
|
|
Invoke-Expression "7z.exe x maven.zip"
|
|
|
|
# add maven to environment variables
|
|
$env:PATH = "$tools\apache-maven-$mavenVer\bin;" + $env:PATH
|
|
$env:M2_HOME = "$tools\apache-maven-$mavenVer"
|
|
$env:MAVEN_OPTS = "-Xmx2g -XX:ReservedCodeCacheSize=1g"
|
|
|
|
Pop-Location
|
|
|
|
# ========================== Hadoop bin package
|
|
# This must match the version at https://github.com/steveloughran/winutils/tree/master/hadoop-2.7.1
|
|
$hadoopVer = "2.7.1"
|
|
$hadoopPath = "$tools\hadoop"
|
|
if (!(Test-Path $hadoopPath)) {
|
|
New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null
|
|
}
|
|
Push-Location $hadoopPath
|
|
|
|
Start-FileDownload "https://github.com/steveloughran/winutils/archive/master.zip" "winutils-master.zip"
|
|
|
|
# extract
|
|
Invoke-Expression "7z.exe x winutils-master.zip"
|
|
|
|
# add hadoop bin to environment variables
|
|
$env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer"
|
|
$env:Path += ";$env:HADOOP_HOME\bin"
|
|
|
|
Pop-Location
|
|
|
|
# ========================== R
|
|
$rVer = "3.6.1"
|
|
$rToolsVer = "3.5.1"
|
|
|
|
InstallR
|
|
InstallRtools
|
|
|
|
$env:R_LIBS_USER = 'c:\RLibrary'
|
|
if ( -not(Test-Path $env:R_LIBS_USER) ) {
|
|
mkdir $env:R_LIBS_USER
|
|
}
|
|
|