[SPARK-29026][SQL] Improve error message in schemaFor
in trait without companion object constructor
### What changes were proposed in this pull request? - For trait without companion object constructor, currently the method to get constructor parameters `constructParams` in `ScalaReflection` will throw exception. ``` scala.ScalaReflectionException: <none> is not a term at scala.reflect.api.Symbols$SymbolApi.asTerm(Symbols.scala:211) at scala.reflect.api.Symbols$SymbolApi.asTerm$(Symbols.scala:211) at scala.reflect.internal.Symbols$SymbolContextApiImpl.asTerm(Symbols.scala:106) at org.apache.spark.sql.catalyst.ScalaReflection.getCompanionConstructor(ScalaReflection.scala:909) at org.apache.spark.sql.catalyst.ScalaReflection.constructParams(ScalaReflection.scala:914) at org.apache.spark.sql.catalyst.ScalaReflection.constructParams$(ScalaReflection.scala:912) at org.apache.spark.sql.catalyst.ScalaReflection$.constructParams(ScalaReflection.scala:47) at org.apache.spark.sql.catalyst.ScalaReflection.getConstructorParameters(ScalaReflection.scala:890) at org.apache.spark.sql.catalyst.ScalaReflection.getConstructorParameters$(ScalaReflection.scala:886) at org.apache.spark.sql.catalyst.ScalaReflection$.getConstructorParameters(ScalaReflection.scala:47) ``` - Instead this PR would throw exception: ``` Unable to find constructor for type [XXX]. This could happen if [XXX] is an interface or a trait without companion object constructor UnsupportedOperationException: ``` In the normal usage of ExpressionEncoder, this can happen if the type is interface extending `scala.Product`. Also, since this is a protected method, this could have been other arbitrary types without constructor. ### Why are the changes needed? - The error message `<none> is not a term` isn't helpful for users to understand the problem. ### Does this PR introduce any user-facing change? - The exception would be thrown instead of runtime exception from the `scala.ScalaReflectionException`. ### How was this patch tested? - Added a unit test to illustrate the `type` where expression encoder will fail and trigger the proposed error message. Closes #25736 from mickjermsurawong-stripe/SPARK-29026. Authored-by: Mick Jermsurawong <mickjermsurawong@stripe.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
This commit is contained in:
parent
54d3f6e7ec
commit
fa75db2059
|
@ -906,7 +906,18 @@ trait ScalaReflection extends Logging {
|
|||
* only defines a constructor via `apply` method.
|
||||
*/
|
||||
private def getCompanionConstructor(tpe: Type): Symbol = {
|
||||
tpe.typeSymbol.asClass.companion.asTerm.typeSignature.member(universe.TermName("apply"))
|
||||
def throwUnsupportedOperation = {
|
||||
throw new UnsupportedOperationException(s"Unable to find constructor for $tpe. " +
|
||||
s"This could happen if $tpe is an interface, or a trait without companion object " +
|
||||
"constructor.")
|
||||
}
|
||||
tpe.typeSymbol.asClass.companion match {
|
||||
case NoSymbol => throwUnsupportedOperation
|
||||
case sym => sym.asTerm.typeSignature.member(universe.TermName("apply")) match {
|
||||
case NoSymbol => throwUnsupportedOperation
|
||||
case constructorSym => constructorSym
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected def constructParams(tpe: Type): Seq[Symbol] = {
|
||||
|
|
|
@ -138,6 +138,14 @@ trait ScroogeLikeExample extends Product1[Int] with Serializable {
|
|||
override def hashCode: Int = x
|
||||
}
|
||||
|
||||
/** Counter-examples to [[ScroogeLikeExample]] as a trait without a companion object constructor */
|
||||
trait TraitProductWithoutCompanion extends Product1[Int] {}
|
||||
|
||||
/** Counter-examples to [[ScroogeLikeExample]] as a trait with no-constructor companion object */
|
||||
object TraitProductWithNoConstructorCompanion {}
|
||||
|
||||
trait TraitProductWithNoConstructorCompanion extends Product1[Int] {}
|
||||
|
||||
class ScalaReflectionSuite extends SparkFunSuite {
|
||||
import org.apache.spark.sql.catalyst.ScalaReflection._
|
||||
|
||||
|
@ -404,6 +412,20 @@ class ScalaReflectionSuite extends SparkFunSuite {
|
|||
StructField("x", IntegerType, nullable = false))), nullable = true))
|
||||
}
|
||||
|
||||
test("SPARK-29026: schemaFor for trait without companion object throws exception ") {
|
||||
val e = intercept[UnsupportedOperationException] {
|
||||
schemaFor[TraitProductWithoutCompanion]
|
||||
}
|
||||
assert(e.getMessage.contains("Unable to find constructor"))
|
||||
}
|
||||
|
||||
test("SPARK-29026: schemaFor for trait with no-constructor companion throws exception ") {
|
||||
val e = intercept[UnsupportedOperationException] {
|
||||
schemaFor[TraitProductWithNoConstructorCompanion]
|
||||
}
|
||||
assert(e.getMessage.contains("Unable to find constructor"))
|
||||
}
|
||||
|
||||
test("SPARK-27625: annotated data types") {
|
||||
assert(serializerFor[FooWithAnnotation].dataType == StructType(Seq(
|
||||
StructField("f1", StringType),
|
||||
|
|
Loading…
Reference in a new issue