SPARK-3177 (on Master Branch)
The JIRA and PR was original created for branch-1.1, and move to master branch now. Chester The Issue is due to that yarn-alpha and yarn have different APIs for certain class fields. In this particular case, the ClientBase using reflection to to address this issue, and we need to different way to test the ClientBase's method. Original ClientBaseSuite using getFieldValue() method to do this. But it doesn't work for yarn-alpha as the API returns an array of String instead of just String (which is the case for Yarn-stable API). To fix the test, I add a new method def getFieldValue2[A: ClassTag, A1: ClassTag, B](clazz: Class[_], field: String, defaults: => B) (mapTo: A => B)(mapTo1: A1 => B) : B = Try(clazz.getField(field)).map(_.get(null)).map { case v: A => mapTo(v) case v1: A1 => mapTo1(v1) case _ => defaults }.toOption.getOrElse(defaults) to handle the cases where the field type can be either type A or A1. In this new method the type A or A1 is pattern matched and corresponding mapTo function (mapTo or mapTo1) is used. Author: chesterxgchen <chester@alpinenow.com> Closes #2204 from chesterxgchen/SPARK-3177-master and squashes the following commits: e72a6ea [chesterxgchen] The Issue is due to that yarn-alpha and yarn have different APIs for certain class fields. In this particular case, the ClientBase using reflection to to address this issue, and we need to different way to test the ClientBase's method. Original ClientBaseSuite using getFieldValue() method to do this. But it doesn't work for yarn-alpha as the API returns an array of String instead of just String (which is the case for Yarn-stable API).
This commit is contained in:
parent
983609a4dd
commit
7d1a37239c
|
@ -38,6 +38,7 @@ import org.scalatest.Matchers
|
||||||
|
|
||||||
import scala.collection.JavaConversions._
|
import scala.collection.JavaConversions._
|
||||||
import scala.collection.mutable.{ HashMap => MutableHashMap }
|
import scala.collection.mutable.{ HashMap => MutableHashMap }
|
||||||
|
import scala.reflect.ClassTag
|
||||||
import scala.util.Try
|
import scala.util.Try
|
||||||
|
|
||||||
import org.apache.spark.{SparkException, SparkConf}
|
import org.apache.spark.{SparkException, SparkConf}
|
||||||
|
@ -200,9 +201,10 @@ class ClientBaseSuite extends FunSuite with Matchers {
|
||||||
|
|
||||||
|
|
||||||
val knownDefMRAppCP: Seq[String] =
|
val knownDefMRAppCP: Seq[String] =
|
||||||
getFieldValue[String, Seq[String]](classOf[MRJobConfig],
|
getFieldValue2[String, Array[String], Seq[String]](
|
||||||
|
classOf[MRJobConfig],
|
||||||
"DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH",
|
"DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH",
|
||||||
Seq[String]())(a => a.split(","))
|
Seq[String]())(a => a.split(","))(a => a.toSeq)
|
||||||
|
|
||||||
val knownYARNAppCP = Some(Seq("/known/yarn/path"))
|
val knownYARNAppCP = Some(Seq("/known/yarn/path"))
|
||||||
|
|
||||||
|
@ -232,6 +234,17 @@ class ClientBaseSuite extends FunSuite with Matchers {
|
||||||
def getFieldValue[A, B](clazz: Class[_], field: String, defaults: => B)(mapTo: A => B): B =
|
def getFieldValue[A, B](clazz: Class[_], field: String, defaults: => B)(mapTo: A => B): B =
|
||||||
Try(clazz.getField(field)).map(_.get(null).asInstanceOf[A]).toOption.map(mapTo).getOrElse(defaults)
|
Try(clazz.getField(field)).map(_.get(null).asInstanceOf[A]).toOption.map(mapTo).getOrElse(defaults)
|
||||||
|
|
||||||
|
def getFieldValue2[A: ClassTag, A1: ClassTag, B](
|
||||||
|
clazz: Class[_],
|
||||||
|
field: String,
|
||||||
|
defaults: => B)(mapTo: A => B)(mapTo1: A1 => B) : B = {
|
||||||
|
Try(clazz.getField(field)).map(_.get(null)).map {
|
||||||
|
case v: A => mapTo(v)
|
||||||
|
case v1: A1 => mapTo1(v1)
|
||||||
|
case _ => defaults
|
||||||
|
}.toOption.getOrElse(defaults)
|
||||||
|
}
|
||||||
|
|
||||||
private class DummyClient(
|
private class DummyClient(
|
||||||
val args: ClientArguments,
|
val args: ClientArguments,
|
||||||
val conf: Configuration,
|
val conf: Configuration,
|
||||||
|
|
Loading…
Reference in a new issue