[SPARK-25036][SQL][FOLLOW-UP] Avoid match may not be exhaustive in Scala-2.12.
## What changes were proposed in this pull request? This is a follow-up pr of #22014. We still have some more compilation errors in scala-2.12 with sbt: ``` [error] [warn] /.../sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala:493: match may not be exhaustive. [error] It would fail on the following input: (_, _) [error] [warn] val typeMatches = (targetType, f.dataType) match { [error] [warn] [error] [warn] /.../sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala:393: match may not be exhaustive. [error] It would fail on the following input: (_, _) [error] [warn] prevBatchOff.get.toStreamProgress(sources).foreach { [error] [warn] [error] [warn] /.../sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala:173: match may not be exhaustive. [error] It would fail on the following input: AggregateExpression(_, _, false, _) [error] [warn] val rewrittenDistinctFunctions = functionsWithDistinct.map { [error] [warn] [error] [warn] /.../sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala:271: match may not be exhaustive. [error] It would fail on the following input: (_, _) [error] [warn] keyWithIndexToValueMetrics.customMetrics.map { [error] [warn] [error] [warn] /.../sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala:959: match may not be exhaustive. [error] It would fail on the following input: CatalogTableType(_) [error] [warn] val tableTypeString = metadata.tableType match { [error] [warn] [error] [warn] /.../sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala:923: match may not be exhaustive. [error] It would fail on the following input: CatalogTableType(_) [error] [warn] hiveTable.setTableType(table.tableType match { [error] [warn] ``` ## How was this patch tested? Manually build with Scala-2.12. Closes #22039 from ueshin/issues/SPARK-25036/fix_match. Authored-by: Takuya UESHIN <ueshin@databricks.com> Signed-off-by: Sean Owen <srowen@gmail.com>
This commit is contained in:
parent
6f6a420078
commit
f62fe435de
|
@ -494,6 +494,8 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
|
|||
case (NumericType, dt) => dt.isInstanceOf[NumericType]
|
||||
case (StringType, dt) => dt == StringType
|
||||
case (BooleanType, dt) => dt == BooleanType
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"$targetType is not matched at fillValue")
|
||||
}
|
||||
// Only fill if the column is part of the cols list.
|
||||
if (typeMatches && cols.exists(col => columnEquals(f.name, col))) {
|
||||
|
|
|
@ -177,6 +177,10 @@ object AggUtils {
|
|||
case agg @ AggregateExpression(aggregateFunction, mode, true, _) =>
|
||||
aggregateFunction.transformDown(distinctColumnAttributeLookup)
|
||||
.asInstanceOf[AggregateFunction]
|
||||
case agg =>
|
||||
throw new IllegalArgumentException(
|
||||
"Non-distinct aggregate is found in functionsWithDistinct " +
|
||||
s"at planAggregateWithOneDistinct: $agg")
|
||||
}
|
||||
|
||||
val partialDistinctAggregate: SparkPlan = {
|
||||
|
|
|
@ -960,6 +960,9 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
|
|||
case EXTERNAL => " EXTERNAL TABLE"
|
||||
case VIEW => " VIEW"
|
||||
case MANAGED => " TABLE"
|
||||
case t =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Unknown table type is found at showCreateHiveTable: $t")
|
||||
}
|
||||
|
||||
builder ++= s"CREATE$tableTypeString ${table.quotedString}"
|
||||
|
|
|
@ -394,6 +394,9 @@ class MicroBatchExecution(
|
|||
case (src: Source, off) => src.commit(off)
|
||||
case (reader: MicroBatchReader, off) =>
|
||||
reader.commit(reader.deserializeOffset(off.json))
|
||||
case (src, _) =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Unknown source is found at constructNextBatch: $src")
|
||||
}
|
||||
} else {
|
||||
throw new IllegalStateException(s"batch ${currentBatchId - 1} doesn't exist")
|
||||
|
|
|
@ -273,6 +273,9 @@ class SymmetricHashJoinStateManager(
|
|||
s.copy(desc = newDesc(desc)) -> value
|
||||
case (s @ StateStoreCustomTimingMetric(_, desc), value) =>
|
||||
s.copy(desc = newDesc(desc)) -> value
|
||||
case (s, _) =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Unknown state store custom metric is found at metrics: $s")
|
||||
}
|
||||
)
|
||||
}
|
||||
|
|
|
@ -927,6 +927,9 @@ private[hive] object HiveClientImpl {
|
|||
case CatalogTableType.MANAGED =>
|
||||
HiveTableType.MANAGED_TABLE
|
||||
case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
|
||||
case t =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Unknown table type is found at toHiveTable: $t")
|
||||
})
|
||||
// Note: In Hive the schema and partition columns must be disjoint sets
|
||||
val (partCols, schema) = table.schema.map(toHiveColumn).partition { c =>
|
||||
|
|
Loading…
Reference in a new issue