ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-24683][K8S] Fix k8s no resource

Browse source 
## What changes were proposed in this pull request?

Make SparkSubmit pass in the main class even if `SparkLauncher.NO_RESOURCE` is the primary resource.

## How was this patch tested?

New integration test written to capture this case.

Author: mcheah <mcheah@palantir.com>

Closes #21660 from mccheah/fix-k8s-no-resource.
This commit is contained in:
mcheah 2018-07-02 10:24:04 -07:00
parent f599cde695
commit 42815548c7
3 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
View file

@ -702,6 +702,8 @@ private[spark] class SparkSubmit extends Logging {
childArgs ++= Array("--primary-java-resource", args.primaryResource)
childArgs ++= Array("--main-class", args.mainClass)
}
} else {
childArgs ++= Array("--main-class", args.mainClass)
}
if (args.childArgs != null) {
args.childArgs.foreach { arg =>

3
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
View file

@ -64,7 +64,8 @@ private[spark] class KubernetesDriverBuilder(
case JavaMainAppResource(_) =>
provideJavaStep(kubernetesConf)
case PythonMainAppResource(_) =>
providePythonStep(kubernetesConf)}.getOrElse(provideJavaStep(kubernetesConf))
providePythonStep(kubernetesConf)}
.getOrElse(provideJavaStep(kubernetesConf))
val allFeatures: Seq[KubernetesFeatureConfigStep] =
(baseFeatures :+ bindingsStep) ++

10
resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
View file

@ -21,17 +21,17 @@ import java.nio.file.{Path, Paths}
import java.util.UUID
import java.util.regex.Pattern
import scala.collection.JavaConverters._
import com.google.common.io.PatternFilenameFilter
import io.fabric8.kubernetes.api.model.{Container, Pod}
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.{Minutes, Seconds, Span}
import scala.collection.JavaConverters._
import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory}
import org.apache.spark.deploy.k8s.integrationtest.config._
import org.apache.spark.launcher.SparkLauncher
private[spark] class KubernetesSuite extends SparkFunSuite
with BeforeAndAfterAll with BeforeAndAfter {
@ -109,6 +109,12 @@ private[spark] class KubernetesSuite extends SparkFunSuite
runSparkPiAndVerifyCompletion()
}
test("Use SparkLauncher.NO_RESOURCE") {
sparkAppConf.setJars(Seq(containerLocalSparkDistroExamplesJar))
runSparkPiAndVerifyCompletion(
appResource = SparkLauncher.NO_RESOURCE)
}
test("Run SparkPi with a master URL without a scheme.") {
val url = kubernetesTestComponents.kubernetesClient.getMasterUrl
val k8sMasterUrl = if (url.getPort < 0) {