[SPARK-16349][SQL] Fall back to isolated class loader when classes not found.
Some Hadoop classes needed by the Hive metastore client jars are not present in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So if the parent class loader fails to find a class, try to load it from the isolated class loader, in case it's available there. Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop libraries and verifying that Spark can talk to the metastore. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #14020 from vanzin/SPARK-16349.
This commit is contained in:
parent
7f38b9d5f4
commit
b4fbe140be
|
@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader(
|
|||
logDebug(s"hive class: $name - ${getResource(classToPath(name))}")
|
||||
super.loadClass(name, resolve)
|
||||
} else {
|
||||
// For shared classes, we delegate to baseClassLoader.
|
||||
// For shared classes, we delegate to baseClassLoader, but fall back in case the
|
||||
// class is not found.
|
||||
logDebug(s"shared class: $name")
|
||||
baseClassLoader.loadClass(name)
|
||||
try {
|
||||
baseClassLoader.loadClass(name)
|
||||
} catch {
|
||||
case _: ClassNotFoundException =>
|
||||
super.loadClass(name, resolve)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader(
|
|||
throw new ClassNotFoundException(
|
||||
s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" +
|
||||
"Please make sure that jars for your version of hive and hadoop are included in the " +
|
||||
s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e)
|
||||
s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e)
|
||||
} else {
|
||||
throw e
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue