ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-9432][SQL] Audit expression unit tests to make sure we pass the proper numeric ranges

Browse source 
JIRA: https://issues.apache.org/jira/browse/SPARK-9432

Author: Yijie Shen <henry.yijieshen@gmail.com>

Closes #7933 from yjshen/numeric_ranges and squashes the following commits:

e719f78 [Yijie Shen] proper integral range check
This commit is contained in:
Yijie Shen 2015-08-04 18:19:26 -07:00 committed by Reynold Xin
parent d92fa14179
commit a7fe48f687
5 changed files with 164 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
View file

@ -23,6 +23,8 @@ import org.apache.spark.sql.types._
class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
import IntegralLiteralTestUtils._
/**
* Runs through the testFunc for all numeric data types.
*
@ -47,6 +49,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Add(Literal.create(null, left.dataType), right), null)
checkEvaluation(Add(left, Literal.create(null, right.dataType)), null)
}
checkEvaluation(Add(positiveShortLit, negativeShortLit), -1.toShort)
checkEvaluation(Add(positiveIntLit, negativeIntLit), -1)
checkEvaluation(Add(positiveLongLit, negativeLongLit), -1L)
}
test("- (UnaryMinus)") {
@ -60,6 +65,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(UnaryMinus(Literal(Int.MinValue)), Int.MinValue)
checkEvaluation(UnaryMinus(Literal(Short.MinValue)), Short.MinValue)
checkEvaluation(UnaryMinus(Literal(Byte.MinValue)), Byte.MinValue)
checkEvaluation(UnaryMinus(positiveShortLit), (- positiveShort).toShort)
checkEvaluation(UnaryMinus(negativeShortLit), (- negativeShort).toShort)
checkEvaluation(UnaryMinus(positiveIntLit), - positiveInt)
checkEvaluation(UnaryMinus(negativeIntLit), - negativeInt)
checkEvaluation(UnaryMinus(positiveLongLit), - positiveLong)
checkEvaluation(UnaryMinus(negativeLongLit), - negativeLong)
}
test("- (Minus)") {
@ -70,6 +81,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Subtract(Literal.create(null, left.dataType), right), null)
checkEvaluation(Subtract(left, Literal.create(null, right.dataType)), null)
}
checkEvaluation(Subtract(positiveShortLit, negativeShortLit),
(positiveShort - negativeShort).toShort)
checkEvaluation(Subtract(positiveIntLit, negativeIntLit), positiveInt - negativeInt)
checkEvaluation(Subtract(positiveLongLit, negativeLongLit), positiveLong - negativeLong)
}
test("* (Multiply)") {
@ -80,6 +95,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Multiply(Literal.create(null, left.dataType), right), null)
checkEvaluation(Multiply(left, Literal.create(null, right.dataType)), null)
}
checkEvaluation(Multiply(positiveShortLit, negativeShortLit),
(positiveShort * negativeShort).toShort)
checkEvaluation(Multiply(positiveIntLit, negativeIntLit), positiveInt * negativeInt)
checkEvaluation(Multiply(positiveLongLit, negativeLongLit), positiveLong * negativeLong)
}
test("/ (Divide) basic") {
@ -99,6 +118,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Divide(Literal(1.toShort), Literal(2.toShort)), 0.toShort)
checkEvaluation(Divide(Literal(1), Literal(2)), 0)
checkEvaluation(Divide(Literal(1.toLong), Literal(2.toLong)), 0.toLong)
checkEvaluation(Divide(positiveShortLit, negativeShortLit), 0.toShort)
checkEvaluation(Divide(positiveIntLit, negativeIntLit), 0)
checkEvaluation(Divide(positiveLongLit, negativeLongLit), 0L)
}
test("/ (Divide) for floating point") {
@ -116,6 +138,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Remainder(left, Literal.create(null, right.dataType)), null)
checkEvaluation(Remainder(left, Literal(convert(0))), null) // mod by 0
}
checkEvaluation(Remainder(positiveShortLit, positiveShortLit), 0.toShort)
checkEvaluation(Remainder(negativeShortLit, negativeShortLit), 0.toShort)
checkEvaluation(Remainder(positiveIntLit, positiveIntLit), 0)
checkEvaluation(Remainder(negativeIntLit, negativeIntLit), 0)
checkEvaluation(Remainder(positiveLongLit, positiveLongLit), 0L)
checkEvaluation(Remainder(negativeLongLit, negativeLongLit), 0L)
}
test("Abs") {
@ -127,6 +155,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Abs(Literal(convert(-1))), convert(1))
checkEvaluation(Abs(Literal.create(null, dataType)), null)
}
checkEvaluation(Abs(positiveShortLit), positiveShort)
checkEvaluation(Abs(negativeShortLit), (- negativeShort).toShort)
checkEvaluation(Abs(positiveIntLit), positiveInt)
checkEvaluation(Abs(negativeIntLit), - negativeInt)
checkEvaluation(Abs(positiveLongLit), positiveLong)
checkEvaluation(Abs(negativeLongLit), - negativeLong)
}
test("MaxOf basic") {
@ -138,6 +172,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(MaxOf(Literal.create(null, small.dataType), large), convert(2))
checkEvaluation(MaxOf(large, Literal.create(null, small.dataType)), convert(2))
}
checkEvaluation(MaxOf(positiveShortLit, negativeShortLit), (positiveShort).toShort)
checkEvaluation(MaxOf(positiveIntLit, negativeIntLit), positiveInt)
checkEvaluation(MaxOf(positiveLongLit, negativeLongLit), positiveLong)
}
test("MaxOf for atomic type") {
@ -156,6 +193,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(MinOf(Literal.create(null, small.dataType), large), convert(2))
checkEvaluation(MinOf(small, Literal.create(null, small.dataType)), convert(1))
}
checkEvaluation(MinOf(positiveShortLit, negativeShortLit), (negativeShort).toShort)
checkEvaluation(MinOf(positiveIntLit, negativeIntLit), negativeInt)
checkEvaluation(MinOf(positiveLongLit, negativeLongLit), negativeLong)
}
test("MinOf for atomic type") {
@ -174,9 +214,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Pmod(left, Literal.create(null, right.dataType)), null)
checkEvaluation(Remainder(left, Literal(convert(0))), null) // mod by 0
}
checkEvaluation(Pmod(-7, 3), 2)
checkEvaluation(Pmod(7.2D, 4.1D), 3.1000000000000005)
checkEvaluation(Pmod(Decimal(0.7), Decimal(0.2)), Decimal(0.1))
checkEvaluation(Pmod(2L, Long.MaxValue), 2L)
checkEvaluation(Pmod(Literal(-7), Literal(3)), 2)
checkEvaluation(Pmod(Literal(7.2D), Literal(4.1D)), 3.1000000000000005)
checkEvaluation(Pmod(Literal(Decimal(0.7)), Literal(Decimal(0.2))), Decimal(0.1))
checkEvaluation(Pmod(Literal(2L), Literal(Long.MaxValue)), 2L)
checkEvaluation(Pmod(positiveShort, negativeShort), positiveShort.toShort)
checkEvaluation(Pmod(positiveInt, negativeInt), positiveInt)
checkEvaluation(Pmod(positiveLong, negativeLong), positiveLong)
}
}

21
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala
View file

@ -23,6 +23,8 @@ import org.apache.spark.sql.types._
class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
import IntegralLiteralTestUtils._
test("BitwiseNOT") {
def check(input: Any, expected: Any): Unit = {
val expr = BitwiseNot(Literal(input))
@ -37,6 +39,12 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
check(123456789123L, ~123456789123L)
checkEvaluation(BitwiseNot(Literal.create(null, IntegerType)), null)
checkEvaluation(BitwiseNot(positiveShortLit), (~positiveShort).toShort)
checkEvaluation(BitwiseNot(negativeShortLit), (~negativeShort).toShort)
checkEvaluation(BitwiseNot(positiveIntLit), ~positiveInt)
checkEvaluation(BitwiseNot(negativeIntLit), ~negativeInt)
checkEvaluation(BitwiseNot(positiveLongLit), ~positiveLong)
checkEvaluation(BitwiseNot(negativeLongLit), ~negativeLong)
}
test("BitwiseAnd") {
@ -56,6 +64,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(BitwiseAnd(nullLit, Literal(1)), null)
checkEvaluation(BitwiseAnd(Literal(1), nullLit), null)
checkEvaluation(BitwiseAnd(nullLit, nullLit), null)
checkEvaluation(BitwiseAnd(positiveShortLit, negativeShortLit),
(positiveShort & negativeShort).toShort)
checkEvaluation(BitwiseAnd(positiveIntLit, negativeIntLit), positiveInt & negativeInt)
checkEvaluation(BitwiseAnd(positiveLongLit, negativeLongLit), positiveLong & negativeLong)
}
test("BitwiseOr") {
@ -75,6 +87,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(BitwiseOr(nullLit, Literal(1)), null)
checkEvaluation(BitwiseOr(Literal(1), nullLit), null)
checkEvaluation(BitwiseOr(nullLit, nullLit), null)
checkEvaluation(BitwiseOr(positiveShortLit, negativeShortLit),
(positiveShort | negativeShort).toShort)
checkEvaluation(BitwiseOr(positiveIntLit, negativeIntLit), positiveInt | negativeInt)
checkEvaluation(BitwiseOr(positiveLongLit, negativeLongLit), positiveLong | negativeLong)
}
test("BitwiseXor") {
@ -94,5 +110,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(BitwiseXor(nullLit, Literal(1)), null)
checkEvaluation(BitwiseXor(Literal(1), nullLit), null)
checkEvaluation(BitwiseXor(nullLit, nullLit), null)
checkEvaluation(BitwiseXor(positiveShortLit, negativeShortLit),
(positiveShort ^ negativeShort).toShort)
checkEvaluation(BitwiseXor(positiveIntLit, negativeIntLit), positiveInt ^ negativeInt)
checkEvaluation(BitwiseXor(positiveLongLit, negativeLongLit), positiveLong ^ negativeLong)
}
}

14
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
View file

@ -28,6 +28,8 @@ import org.apache.spark.unsafe.types.CalendarInterval
class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
import IntegralLiteralTestUtils._
val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val sdfDate = new SimpleDateFormat("yyyy-MM-dd")
val d = new Date(sdf.parse("2015-04-08 13:10:15").getTime)
@ -212,6 +214,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
null)
checkEvaluation(DateAdd(Literal.create(null, DateType), Literal.create(null, IntegerType)),
null)
checkEvaluation(
DateAdd(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 49627)
checkEvaluation(
DateAdd(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -15910)
}
test("date_sub") {
@ -226,6 +232,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
null)
checkEvaluation(DateSub(Literal.create(null, DateType), Literal.create(null, IntegerType)),
null)
checkEvaluation(
DateSub(Literal(Date.valueOf("2016-02-28")), positiveIntLit), -15909)
checkEvaluation(
DateSub(Literal(Date.valueOf("2016-02-28")), negativeIntLit), 49628)
}
test("time_add") {
@ -282,6 +292,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
null)
checkEvaluation(
AddMonths(Literal(Date.valueOf("2015-01-30")), Literal(Int.MinValue)), -7293498)
checkEvaluation(
AddMonths(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 1014213)
checkEvaluation(
AddMonths(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -980528)
}
test("months_between") {

42
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntegralLiteralTestUtils.scala Normal file
View file

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.sql.catalyst.expressions
/**
* Utilities to make sure we pass the proper numeric ranges
*/
object IntegralLiteralTestUtils {
val positiveShort: Short = (Byte.MaxValue + 1).toShort
val negativeShort: Short = (Byte.MinValue - 1).toShort
val positiveShortLit: Literal = Literal(positiveShort)
val negativeShortLit: Literal = Literal(negativeShort)
val positiveInt: Int = Short.MaxValue + 1
val negativeInt: Int = Short.MinValue - 1
val positiveIntLit: Literal = Literal(positiveInt)
val negativeIntLit: Literal = Literal(negativeInt)
val positiveLong: Long = Int.MaxValue + 1L
val negativeLong: Long = Int.MinValue - 1L
val positiveLongLit: Literal = Literal(positiveLong)
val negativeLongLit: Literal = Literal(negativeLong)
}

40
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala
View file

@ -30,6 +30,8 @@ import org.apache.spark.sql.types._
class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
import IntegralLiteralTestUtils._
/**
* Used for testing leaf math expressions.
*
@ -293,6 +295,9 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Bin(l3), java.lang.Long.toBinaryString(123), row)
checkEvaluation(Bin(l4), java.lang.Long.toBinaryString(1234), row)
checkEvaluation(Bin(l5), java.lang.Long.toBinaryString(-123), row)
checkEvaluation(Bin(positiveLongLit), java.lang.Long.toBinaryString(positiveLong))
checkEvaluation(Bin(negativeLongLit), java.lang.Long.toBinaryString(negativeLong))
}
test("log2") {
@ -324,6 +329,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(ShiftLeft(Literal(21.toLong), Literal(1)), 42.toLong)
checkEvaluation(ShiftLeft(Literal(-21.toLong), Literal(1)), -42.toLong)
checkEvaluation(ShiftLeft(positiveIntLit, positiveIntLit), positiveInt << positiveInt)
checkEvaluation(ShiftLeft(positiveIntLit, negativeIntLit), positiveInt << negativeInt)
checkEvaluation(ShiftLeft(negativeIntLit, positiveIntLit), negativeInt << positiveInt)
checkEvaluation(ShiftLeft(negativeIntLit, negativeIntLit), negativeInt << negativeInt)
checkEvaluation(ShiftLeft(positiveLongLit, positiveIntLit), positiveLong << positiveInt)
checkEvaluation(ShiftLeft(positiveLongLit, negativeIntLit), positiveLong << negativeInt)
checkEvaluation(ShiftLeft(negativeLongLit, positiveIntLit), negativeLong << positiveInt)
checkEvaluation(ShiftLeft(negativeLongLit, negativeIntLit), negativeLong << negativeInt)
}
test("shift right") {
@ -335,6 +349,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(ShiftRight(Literal(42.toLong), Literal(1)), 21.toLong)
checkEvaluation(ShiftRight(Literal(-42.toLong), Literal(1)), -21.toLong)
checkEvaluation(ShiftRight(positiveIntLit, positiveIntLit), positiveInt >> positiveInt)
checkEvaluation(ShiftRight(positiveIntLit, negativeIntLit), positiveInt >> negativeInt)
checkEvaluation(ShiftRight(negativeIntLit, positiveIntLit), negativeInt >> positiveInt)
checkEvaluation(ShiftRight(negativeIntLit, negativeIntLit), negativeInt >> negativeInt)
checkEvaluation(ShiftRight(positiveLongLit, positiveIntLit), positiveLong >> positiveInt)
checkEvaluation(ShiftRight(positiveLongLit, negativeIntLit), positiveLong >> negativeInt)
checkEvaluation(ShiftRight(negativeLongLit, positiveIntLit), negativeLong >> positiveInt)
checkEvaluation(ShiftRight(negativeLongLit, negativeIntLit), negativeLong >> negativeInt)
}
test("shift right unsigned") {
@ -346,6 +369,23 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(ShiftRightUnsigned(Literal(42.toLong), Literal(1)), 21.toLong)
checkEvaluation(ShiftRightUnsigned(Literal(-42.toLong), Literal(1)), 9223372036854775787L)
checkEvaluation(ShiftRightUnsigned(positiveIntLit, positiveIntLit),
positiveInt >>> positiveInt)
checkEvaluation(ShiftRightUnsigned(positiveIntLit, negativeIntLit),
positiveInt >>> negativeInt)
checkEvaluation(ShiftRightUnsigned(negativeIntLit, positiveIntLit),
negativeInt >>> positiveInt)
checkEvaluation(ShiftRightUnsigned(negativeIntLit, negativeIntLit),
negativeInt >>> negativeInt)
checkEvaluation(ShiftRightUnsigned(positiveLongLit, positiveIntLit),
positiveLong >>> positiveInt)
checkEvaluation(ShiftRightUnsigned(positiveLongLit, negativeIntLit),
positiveLong >>> negativeInt)
checkEvaluation(ShiftRightUnsigned(negativeLongLit, positiveIntLit),
negativeLong >>> positiveInt)
checkEvaluation(ShiftRightUnsigned(negativeLongLit, negativeIntLit),
negativeLong >>> negativeInt)
}
test("hex") {