[SPARK-24544][SQL] Print actual failure cause when look up function failed

## What changes were proposed in this pull request?

When we operate as below:
`
0: jdbc:hive2://xxx/> create  function funnel_analysis as 'com.xxx.hive.extend.udf.UapFunnelAnalysis';
`

`
0: jdbc:hive2://xxx/> select funnel_analysis(1,",",1,'');
Error: org.apache.spark.sql.AnalysisException: Undefined function: 'funnel_analysis'. This function is neither a registered temporary function nor a permanent function registered in the database 'xxx'.; line 1 pos 7 (state=,code=0)
`

`
0: jdbc:hive2://xxx/> describe function funnel_analysis;
+-----------------------------------------------------------+--+
|                       function_desc                       |
+-----------------------------------------------------------+--+
| Function: xxx.funnel_analysis                            |
| Class: com.xxx.hive.extend.udf.UapFunnelAnalysis  |
| Usage: N/A.                                               |
+-----------------------------------------------------------+--+
`
We can see describe funtion will get right information,but when we actually use this funtion,we will get an undefined exception.
Which is really misleading,the real cause is below:
 `
No handler for Hive UDF 'com.xxx.xxx.hive.extend.udf.UapFunnelAnalysis': java.lang.IllegalStateException: Should not be called directly;
	at org.apache.hadoop.hive.ql.udf.generic.GenericUDTF.initialize(GenericUDTF.java:72)
	at org.apache.spark.sql.hive.HiveGenericUDTF.outputInspector$lzycompute(hiveUDFs.scala:204)
	at org.apache.spark.sql.hive.HiveGenericUDTF.outputInspector(hiveUDFs.scala:204)
	at org.apache.spark.sql.hive.HiveGenericUDTF.elementSchema$lzycompute(hiveUDFs.scala:212)
	at org.apache.spark.sql.hive.HiveGenericUDTF.elementSchema(hiveUDFs.scala:212)
`
This patch print the actual failure for quick debugging.
## How was this patch tested?
UT

Closes #21790 from caneGuy/zhoukang/print-warning1.

Authored-by: zhoukang <zhoukang199191@gmail.com>
Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
zhoukang 2019-01-01 09:13:13 -06:00 committed by Sean Owen
parent 5f0ddd2d6e
commit 2bf4d97118
4 changed files with 25 additions and 7 deletions

View file

@ -40,10 +40,10 @@ class NoSuchPartitionException(
class NoSuchPermanentFunctionException(db: String, func: String)
extends AnalysisException(s"Function '$func' not found in database '$db'")
class NoSuchFunctionException(db: String, func: String)
class NoSuchFunctionException(db: String, func: String, cause: Option[Throwable] = None)
extends AnalysisException(
s"Undefined function: '$func'. This function is neither a registered temporary function nor " +
s"a permanent function registered in the database '$db'.")
s"a permanent function registered in the database '$db'.", cause = cause)
class NoSuchPartitionsException(db: String, table: String, specs: Seq[TablePartitionSpec])
extends AnalysisException(

View file

@ -1222,9 +1222,10 @@ class SessionCatalog(
databaseExists(db) && externalCatalog.functionExists(db, name.funcName)
}
protected def failFunctionLookup(name: FunctionIdentifier): Nothing = {
protected[sql] def failFunctionLookup(
name: FunctionIdentifier, cause: Option[Throwable] = None): Nothing = {
throw new NoSuchFunctionException(
db = name.database.getOrElse(getCurrentDatabase), func = name.funcName)
db = name.database.getOrElse(getCurrentDatabase), func = name.funcName, cause)
}
/**

View file

@ -1448,4 +1448,18 @@ abstract class SessionCatalogSuite extends AnalysisTest {
}
}
}
test("SPARK-24544: test print actual failure cause when look up function failed") {
withBasicCatalog { catalog =>
val cause = intercept[NoSuchFunctionException] {
catalog.failFunctionLookup(FunctionIdentifier("failureFunc"),
Some(new Exception("Actual error")))
}
// fullStackTrace will be printed, but `cause.getMessage` has been
// override in `AnalysisException`,so here we get the root cause
// exception message for check.
assert(cause.cause.get.getMessage.contains("Actual error"))
}
}
}

View file

@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DecimalType, DoubleType}
import org.apache.spark.util.Utils
private[sql] class HiveSessionCatalog(
@ -141,8 +142,10 @@ private[sql] class HiveSessionCatalog(
// let's try to load it as a Hive's built-in function.
// Hive is case insensitive.
val functionName = funcName.unquotedString.toLowerCase(Locale.ROOT)
logWarning("Encountered a failure during looking up function:" +
s" ${Utils.exceptionString(error)}")
if (!hiveFunctions.contains(functionName)) {
failFunctionLookup(funcName)
failFunctionLookup(funcName, Some(error))
}
// TODO: Remove this fallback path once we implement the list of fallback functions
@ -150,12 +153,12 @@ private[sql] class HiveSessionCatalog(
val functionInfo = {
try {
Option(HiveFunctionRegistry.getFunctionInfo(functionName)).getOrElse(
failFunctionLookup(funcName))
failFunctionLookup(funcName, Some(error)))
} catch {
// If HiveFunctionRegistry.getFunctionInfo throws an exception,
// we are failing to load a Hive builtin function, which means that
// the given function is not a Hive builtin function.
case NonFatal(e) => failFunctionLookup(funcName)
case NonFatal(e) => failFunctionLookup(funcName, Some(e))
}
}
val className = functionInfo.getFunctionClass.getName