[SPARK-19949][SQL][FOLLOW-UP] move FailureSafeParser from catalyst to sql core
## What changes were proposed in this pull request? The `FailureSafeParser` is only used in sql core, it doesn't make sense to put it in catalyst module. ## How was this patch tested? N/A Author: Wenchen Fan <wenchen@databricks.com> Closes #17408 from cloud-fan/minor.
This commit is contained in:
parent
be85245a98
commit
0b903caef3
|
@ -0,0 +1,33 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.spark.sql.catalyst.util
|
||||||
|
|
||||||
|
import org.apache.spark.sql.catalyst.InternalRow
|
||||||
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exception thrown when the underlying parser meet a bad record and can't parse it.
|
||||||
|
* @param record a function to return the record that cause the parser to fail
|
||||||
|
* @param partialResult a function that returns an optional row, which is the partial result of
|
||||||
|
* parsing this bad record.
|
||||||
|
* @param cause the actual exception about why the record is bad and can't be parsed.
|
||||||
|
*/
|
||||||
|
case class BadRecordException(
|
||||||
|
record: () => UTF8String,
|
||||||
|
partialResult: () => Option[InternalRow],
|
||||||
|
cause: Throwable) extends Exception(cause)
|
|
@ -27,11 +27,10 @@ import org.apache.spark.Partition
|
||||||
import org.apache.spark.annotation.InterfaceStability
|
import org.apache.spark.annotation.InterfaceStability
|
||||||
import org.apache.spark.rdd.RDD
|
import org.apache.spark.rdd.RDD
|
||||||
import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
|
import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
|
||||||
import org.apache.spark.sql.catalyst.util.FailureSafeParser
|
|
||||||
import org.apache.spark.sql.execution.LogicalRDD
|
import org.apache.spark.sql.execution.LogicalRDD
|
||||||
import org.apache.spark.sql.execution.command.DDLUtils
|
import org.apache.spark.sql.execution.command.DDLUtils
|
||||||
|
import org.apache.spark.sql.execution.datasources.{DataSource, FailureSafeParser}
|
||||||
import org.apache.spark.sql.execution.datasources.csv._
|
import org.apache.spark.sql.execution.datasources.csv._
|
||||||
import org.apache.spark.sql.execution.datasources.DataSource
|
|
||||||
import org.apache.spark.sql.execution.datasources.jdbc._
|
import org.apache.spark.sql.execution.datasources.jdbc._
|
||||||
import org.apache.spark.sql.execution.datasources.json.TextInputJsonDataSource
|
import org.apache.spark.sql.execution.datasources.json.TextInputJsonDataSource
|
||||||
import org.apache.spark.sql.types.{StringType, StructType}
|
import org.apache.spark.sql.types.{StringType, StructType}
|
||||||
|
|
|
@ -15,10 +15,11 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.sql.catalyst.util
|
package org.apache.spark.sql.execution.datasources
|
||||||
|
|
||||||
import org.apache.spark.sql.catalyst.InternalRow
|
import org.apache.spark.sql.catalyst.InternalRow
|
||||||
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
|
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
|
||||||
|
import org.apache.spark.sql.catalyst.util._
|
||||||
import org.apache.spark.sql.types.StructType
|
import org.apache.spark.sql.types.StructType
|
||||||
import org.apache.spark.unsafe.types.UTF8String
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
||||||
|
@ -69,15 +70,3 @@ class FailureSafeParser[IN](
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Exception thrown when the underlying parser meet a bad record and can't parse it.
|
|
||||||
* @param record a function to return the record that cause the parser to fail
|
|
||||||
* @param partialResult a function that returns an optional row, which is the partial result of
|
|
||||||
* parsing this bad record.
|
|
||||||
* @param cause the actual exception about why the record is bad and can't be parsed.
|
|
||||||
*/
|
|
||||||
case class BadRecordException(
|
|
||||||
record: () => UTF8String,
|
|
||||||
partialResult: () => Option[InternalRow],
|
|
||||||
cause: Throwable) extends Exception(cause)
|
|
|
@ -30,7 +30,8 @@ import com.univocity.parsers.csv.CsvParser
|
||||||
import org.apache.spark.internal.Logging
|
import org.apache.spark.internal.Logging
|
||||||
import org.apache.spark.sql.catalyst.InternalRow
|
import org.apache.spark.sql.catalyst.InternalRow
|
||||||
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
|
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
|
||||||
import org.apache.spark.sql.catalyst.util.{BadRecordException, DateTimeUtils, FailureSafeParser}
|
import org.apache.spark.sql.catalyst.util.{BadRecordException, DateTimeUtils}
|
||||||
|
import org.apache.spark.sql.execution.datasources.FailureSafeParser
|
||||||
import org.apache.spark.sql.types._
|
import org.apache.spark.sql.types._
|
||||||
import org.apache.spark.unsafe.types.UTF8String
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
||||||
|
|
|
@ -33,8 +33,7 @@ import org.apache.spark.rdd.{BinaryFileRDD, RDD}
|
||||||
import org.apache.spark.sql.{AnalysisException, Dataset, Encoders, SparkSession}
|
import org.apache.spark.sql.{AnalysisException, Dataset, Encoders, SparkSession}
|
||||||
import org.apache.spark.sql.catalyst.InternalRow
|
import org.apache.spark.sql.catalyst.InternalRow
|
||||||
import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
|
import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
|
||||||
import org.apache.spark.sql.catalyst.util.FailureSafeParser
|
import org.apache.spark.sql.execution.datasources._
|
||||||
import org.apache.spark.sql.execution.datasources.{CodecStreams, DataSource, HadoopFileLinesReader, PartitionedFile}
|
|
||||||
import org.apache.spark.sql.execution.datasources.text.TextFileFormat
|
import org.apache.spark.sql.execution.datasources.text.TextFileFormat
|
||||||
import org.apache.spark.sql.types.StructType
|
import org.apache.spark.sql.types.StructType
|
||||||
import org.apache.spark.unsafe.types.UTF8String
|
import org.apache.spark.unsafe.types.UTF8String
|
||||||
|
|
Loading…
Reference in a new issue