ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-22654][TESTS] Retry Spark tarball download if failed in HiveExternalCatalogVersionsSuite

Browse source 
## What changes were proposed in this pull request?

Adds a simple loop to retry download of Spark tarballs from different mirrors if the download fails.

## How was this patch tested?

Existing tests

Author: Sean Owen <sowen@cloudera.com>

Closes #19851 from srowen/SPARK-22654.
This commit is contained in:
Sean Owen 2017-12-01 01:21:52 +09:00 committed by hyukjinkwon
parent 9c29c55763
commit 6eb203fae7
1 changed files with 19 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
View file

@ -20,6 +20,8 @@ package org.apache.spark.sql.hive
import java.io.File
import java.nio.file.Files
import scala.sys.process._
import org.apache.spark.TestUtils
import org.apache.spark.sql.{QueryTest, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
@ -50,14 +52,24 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
super.afterAll()
}
private def tryDownloadSpark(version: String, path: String): Unit = {
// Try mirrors a few times until one succeeds
for (i <- 0 until 3) {
val preferredMirror =
Seq("wget", "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim
val url = s"$preferredMirror/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz"
logInfo(s"Downloading Spark $version from $url")
if (Seq("wget", url, "-q", "-P", path).! == 0) {
return
}
logWarning(s"Failed to download Spark $version from $url")
}
fail(s"Unable to download Spark $version")
}
private def downloadSpark(version: String): Unit = {
import scala.sys.process._
val preferredMirror =
Seq("wget", "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim
val url = s"$preferredMirror/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz"
Seq("wget", url, "-q", "-P", sparkTestingDir.getCanonicalPath).!
tryDownloadSpark(version, sparkTestingDir.getCanonicalPath)
val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath
val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath