ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-13601] [TESTS] use 1 partition in tests to avoid race conditions

Browse source 
## What changes were proposed in this pull request?

Fix race conditions when cleanup files.

## How was this patch tested?

Existing tests.

Author: Davies Liu <davies@databricks.com>

Closes #11507 from davies/flaky.
This commit is contained in:
Davies Liu 2016-03-03 17:46:28 -08:00 committed by Davies Liu
parent b373a88862
commit d062587dd2
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
sql/hive/src/test/scala/org/apache/spark/sql/sources/CommitFailureTestRelationSuite.scala
View file

@ -51,7 +51,7 @@ class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton
withTempPath { file => withTempPath { file =>
// fail the job in the middle of writing // fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)}) val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = sqlContext.range(0, 10).select(divideByZero(col("id"))) val df = sqlContext.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false SimpleTextRelation.callbackCalled = false
intercept[SparkException] { intercept[SparkException] {
@ -69,7 +69,8 @@ class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton
withTempPath { file => withTempPath { file =>
// fail the job in the middle of writing // fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)}) val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = sqlContext.range(0, 10).select(col("id").mod(2).as("key"), divideByZero(col("id"))) val df = sqlContext.range(0, 10).coalesce(1)
.select(col("id").mod(2).as("key"), divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false SimpleTextRelation.callbackCalled = false
intercept[SparkException] { intercept[SparkException] {
@ -87,7 +88,7 @@ class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton
SimpleTextRelation.failCommitter = false SimpleTextRelation.failCommitter = false
withTempPath { file => withTempPath { file =>
// fail the job in the middle of writing // fail the job in the middle of writing
val df = sqlContext.range(0, 10).select(col("id").mod(2).as("key"), col("id")) val df = sqlContext.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true SimpleTextRelation.failWriter = true