[SPARK-4504][Examples] fix run-example failure if multiple assembly jars exist
Fix run-example script to fail fast with useful error message if multiple example assembly JARs are present. Author: Venkata Ramana Gollamudi <ramana.gollamudi@huawei.com> Closes #3377 from gvramana/run-example_fails and squashes the following commits: fa7f481 [Venkata Ramana Gollamudi] Fixed review comments, avoiding ls output scanning. 6aa1ab7 [Venkata Ramana Gollamudi] Fix run-examples script error during multiple jars
This commit is contained in:
parent
2604bc35d7
commit
74de94ea6d
|
@ -72,21 +72,24 @@ else
|
|||
assembly_folder="$ASSEMBLY_DIR"
|
||||
fi
|
||||
|
||||
num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar$" | wc -l)"
|
||||
if [ "$num_jars" -eq "0" ]; then
|
||||
echo "Failed to find Spark assembly in $assembly_folder"
|
||||
echo "You need to build Spark before running this program."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$num_jars" -gt "1" ]; then
|
||||
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar$")
|
||||
echo "Found multiple Spark assembly jars in $assembly_folder:"
|
||||
echo "$jars_list"
|
||||
echo "Please remove all but one jar."
|
||||
exit 1
|
||||
fi
|
||||
num_jars=0
|
||||
|
||||
ASSEMBLY_JAR="$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)"
|
||||
for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do
|
||||
if [[ ! -e "$f" ]]; then
|
||||
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
|
||||
echo "You need to build Spark before running this program." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
ASSEMBLY_JAR="$f"
|
||||
num_jars=$((num_jars+1))
|
||||
done
|
||||
|
||||
if [ "$num_jars" -gt "1" ]; then
|
||||
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
|
||||
ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2
|
||||
echo "Please remove all but one jar." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify that versions of java used to build the jars and run Spark are compatible
|
||||
jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1)
|
||||
|
|
|
@ -35,17 +35,32 @@ else
|
|||
fi
|
||||
|
||||
if [ -f "$FWDIR/RELEASE" ]; then
|
||||
export SPARK_EXAMPLES_JAR="`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`"
|
||||
elif [ -e "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
|
||||
export SPARK_EXAMPLES_JAR="`ls "$EXAMPLES_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-examples-*hadoop*.jar`"
|
||||
JAR_PATH="${FWDIR}/lib"
|
||||
else
|
||||
JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}"
|
||||
fi
|
||||
|
||||
if [[ -z "$SPARK_EXAMPLES_JAR" ]]; then
|
||||
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
|
||||
echo "You need to build Spark before running this program" 1>&2
|
||||
JAR_COUNT=0
|
||||
|
||||
for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do
|
||||
if [[ ! -e "$f" ]]; then
|
||||
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
|
||||
echo "You need to build Spark before running this program" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
SPARK_EXAMPLES_JAR="$f"
|
||||
JAR_COUNT=$((JAR_COUNT+1))
|
||||
done
|
||||
|
||||
if [ "$JAR_COUNT" -gt "1" ]; then
|
||||
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
|
||||
ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2
|
||||
echo "Please remove all but one jar." 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export SPARK_EXAMPLES_JAR
|
||||
|
||||
EXAMPLE_MASTER=${MASTER:-"local[*]"}
|
||||
|
||||
if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then
|
||||
|
|
Loading…
Reference in a new issue