[SPARK-16349][SQL] Fall back to isolated class loader when classes not found.
Some Hadoop classes needed by the Hive metastore client jars are not present in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So if the parent class loader fails to find a class, try to load it from the isolated class loader, in case it's available there. Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop libraries and verifying that Spark can talk to the metastore. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #14020 from vanzin/SPARK-16349.
This commit is contained in:
parent
7f38b9d5f4
commit
b4fbe140be
|
@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader(
|
||||||
logDebug(s"hive class: $name - ${getResource(classToPath(name))}")
|
logDebug(s"hive class: $name - ${getResource(classToPath(name))}")
|
||||||
super.loadClass(name, resolve)
|
super.loadClass(name, resolve)
|
||||||
} else {
|
} else {
|
||||||
// For shared classes, we delegate to baseClassLoader.
|
// For shared classes, we delegate to baseClassLoader, but fall back in case the
|
||||||
|
// class is not found.
|
||||||
logDebug(s"shared class: $name")
|
logDebug(s"shared class: $name")
|
||||||
baseClassLoader.loadClass(name)
|
try {
|
||||||
|
baseClassLoader.loadClass(name)
|
||||||
|
} catch {
|
||||||
|
case _: ClassNotFoundException =>
|
||||||
|
super.loadClass(name, resolve)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader(
|
||||||
throw new ClassNotFoundException(
|
throw new ClassNotFoundException(
|
||||||
s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" +
|
s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" +
|
||||||
"Please make sure that jars for your version of hive and hadoop are included in the " +
|
"Please make sure that jars for your version of hive and hadoop are included in the " +
|
||||||
s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e)
|
s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e)
|
||||||
} else {
|
} else {
|
||||||
throw e
|
throw e
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue