4a377aff2d
This patch will bring support for broadcasting objects larger than 2G. pickle, zlib, FrameSerializer and Array[Byte] all can not support objects larger than 2G, so this patch introduce LargeObjectSerializer to serialize broadcast objects, the object will be serialized and compressed into small chunks, it also change the type of Broadcast[Array[Byte]]] into Broadcast[Array[Array[Byte]]]]. Testing for support broadcast objects larger than 2G is slow and memory hungry, so this is tested manually, could be added into SparkPerf. Author: Davies Liu <davies@databricks.com> Author: Davies Liu <davies.liu@gmail.com> Closes #2659 from davies/huge and squashes the following commits: 7b57a14 [Davies Liu] add more tests for broadcast 28acff9 [Davies Liu] Merge branch 'master' of github.com:apache/spark into huge a2f6a02 [Davies Liu] bug fix 4820613 [Davies Liu] Merge branch 'master' of github.com:apache/spark into huge 5875c73 [Davies Liu] address comments 10a349b [Davies Liu] address comments 0c33016 [Davies Liu] Merge branch 'master' of github.com:apache/spark into huge 6182c8f [Davies Liu] Merge branch 'master' into huge d94b68f [Davies Liu] Merge branch 'master' of github.com:apache/spark into huge 2514848 [Davies Liu] address comments fda395b [Davies Liu] Merge branch 'master' of github.com:apache/spark into huge 1c2d928 [Davies Liu] fix scala style 091b107 [Davies Liu] broadcast objects larger than 2G
126 lines
3.6 KiB
Bash
Executable file
126 lines
3.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
|
|
# Figure out where the Spark framework is installed
|
|
FWDIR="$(cd "`dirname "$0"`"; cd ../; pwd)"
|
|
|
|
# CD into the python directory to find things on the right path
|
|
cd "$FWDIR/python"
|
|
|
|
FAILED=0
|
|
LOG_FILE=unit-tests.log
|
|
|
|
rm -f $LOG_FILE
|
|
|
|
# Remove the metastore and warehouse directory created by the HiveContext tests in Spark SQL
|
|
rm -rf metastore warehouse
|
|
|
|
function run_test() {
|
|
echo "Running test: $1" | tee -a $LOG_FILE
|
|
|
|
SPARK_TESTING=1 time "$FWDIR"/bin/pyspark $1 2>&1 | tee -a $LOG_FILE
|
|
|
|
FAILED=$((PIPESTATUS[0]||$FAILED))
|
|
|
|
# Fail and exit on the first test failure.
|
|
if [[ $FAILED != 0 ]]; then
|
|
cat $LOG_FILE | grep -v "^[0-9][0-9]*" # filter all lines starting with a number.
|
|
echo -en "\033[31m" # Red
|
|
echo "Had test failures; see logs."
|
|
echo -en "\033[0m" # No color
|
|
exit -1
|
|
fi
|
|
}
|
|
|
|
function run_core_tests() {
|
|
echo "Run core tests ..."
|
|
run_test "pyspark/rdd.py"
|
|
run_test "pyspark/context.py"
|
|
run_test "pyspark/conf.py"
|
|
PYSPARK_DOC_TEST=1 run_test "pyspark/broadcast.py"
|
|
PYSPARK_DOC_TEST=1 run_test "pyspark/accumulators.py"
|
|
run_test "pyspark/serializers.py"
|
|
run_test "pyspark/shuffle.py"
|
|
run_test "pyspark/tests.py"
|
|
}
|
|
|
|
function run_sql_tests() {
|
|
echo "Run sql tests ..."
|
|
run_test "pyspark/sql.py"
|
|
}
|
|
|
|
function run_mllib_tests() {
|
|
echo "Run mllib tests ..."
|
|
run_test "pyspark/mllib/classification.py"
|
|
run_test "pyspark/mllib/clustering.py"
|
|
run_test "pyspark/mllib/feature.py"
|
|
run_test "pyspark/mllib/linalg.py"
|
|
run_test "pyspark/mllib/rand.py"
|
|
run_test "pyspark/mllib/recommendation.py"
|
|
run_test "pyspark/mllib/regression.py"
|
|
run_test "pyspark/mllib/stat.py"
|
|
run_test "pyspark/mllib/tree.py"
|
|
run_test "pyspark/mllib/util.py"
|
|
run_test "pyspark/mllib/tests.py"
|
|
}
|
|
|
|
function run_streaming_tests() {
|
|
echo "Run streaming tests ..."
|
|
run_test "pyspark/streaming/util.py"
|
|
run_test "pyspark/streaming/tests.py"
|
|
}
|
|
|
|
echo "Running PySpark tests. Output is in python/$LOG_FILE."
|
|
|
|
export PYSPARK_PYTHON="python"
|
|
|
|
# Try to test with Python 2.6, since that's the minimum version that we support:
|
|
if [ $(which python2.6) ]; then
|
|
export PYSPARK_PYTHON="python2.6"
|
|
fi
|
|
|
|
echo "Testing with Python version:"
|
|
$PYSPARK_PYTHON --version
|
|
|
|
run_core_tests
|
|
run_sql_tests
|
|
run_mllib_tests
|
|
run_streaming_tests
|
|
|
|
# Try to test with PyPy
|
|
if [ $(which pypy) ]; then
|
|
export PYSPARK_PYTHON="pypy"
|
|
echo "Testing with PyPy version:"
|
|
$PYSPARK_PYTHON --version
|
|
|
|
run_core_tests
|
|
run_sql_tests
|
|
run_streaming_tests
|
|
fi
|
|
|
|
if [[ $FAILED == 0 ]]; then
|
|
echo -en "\033[32m" # Green
|
|
echo "Tests passed."
|
|
echo -en "\033[0m" # No color
|
|
fi
|
|
|
|
# TODO: in the long-run, it would be nice to use a test runner like `nose`.
|
|
# The doctest fixtures are the current barrier to doing this.
|