[SPARK-12568][SQL] Add BINARY to Encoders
Author: Michael Armbrust <michael@databricks.com> Closes #10516 from marmbrus/datasetCleanup.
This commit is contained in:
parent
7058dc1150
commit
53beddc5bf
|
@ -157,6 +157,12 @@ object Encoders {
|
|||
*/
|
||||
def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder()
|
||||
|
||||
/**
|
||||
* An encoder for arrays of bytes.
|
||||
* @since 1.6.1
|
||||
*/
|
||||
def BINARY: Encoder[Array[Byte]] = ExpressionEncoder()
|
||||
|
||||
/**
|
||||
* Creates an encoder for Java Bean of type T.
|
||||
*
|
||||
|
|
|
@ -198,6 +198,15 @@ case class ExpressionEncoder[T](
|
|||
@transient
|
||||
private lazy val constructProjection = GenerateSafeProjection.generate(fromRowExpression :: Nil)
|
||||
|
||||
/**
|
||||
* Returns this encoder where it has been bound to its own output (i.e. no remaping of columns
|
||||
* is performed).
|
||||
*/
|
||||
def defaultBinding: ExpressionEncoder[T] = {
|
||||
val attrs = schema.toAttributes
|
||||
resolve(attrs, OuterScopes.outerScopes).bind(attrs)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an encoded version of `t` as a Spark SQL row. Note that multiple calls to
|
||||
* toRow are allowed to return the same actual [[InternalRow]] object. Thus, the caller should
|
||||
|
|
|
@ -77,6 +77,8 @@ class JavaSerializable(val value: Int) extends Serializable {
|
|||
}
|
||||
|
||||
class ExpressionEncoderSuite extends SparkFunSuite {
|
||||
OuterScopes.outerScopes.put(getClass.getName, this)
|
||||
|
||||
implicit def encoder[T : TypeTag]: ExpressionEncoder[T] = ExpressionEncoder()
|
||||
|
||||
// test flat encoders
|
||||
|
@ -278,8 +280,6 @@ class ExpressionEncoderSuite extends SparkFunSuite {
|
|||
}
|
||||
}
|
||||
|
||||
private val outers: ConcurrentMap[String, AnyRef] = new MapMaker().weakValues().makeMap()
|
||||
outers.put(getClass.getName, this)
|
||||
private def encodeDecodeTest[T : ExpressionEncoder](
|
||||
input: T,
|
||||
testName: String): Unit = {
|
||||
|
@ -287,7 +287,7 @@ class ExpressionEncoderSuite extends SparkFunSuite {
|
|||
val encoder = implicitly[ExpressionEncoder[T]]
|
||||
val row = encoder.toRow(input)
|
||||
val schema = encoder.schema.toAttributes
|
||||
val boundEncoder = encoder.resolve(schema, outers).bind(schema)
|
||||
val boundEncoder = encoder.defaultBinding
|
||||
val convertedBack = try boundEncoder.fromRow(row) catch {
|
||||
case e: Exception =>
|
||||
fail(
|
||||
|
|
Loading…
Reference in a new issue