From 26cd2ca0402d7d49780116d45a5622a45c79f661 Mon Sep 17 00:00:00 2001 From: Burak Yavuz Date: Tue, 18 Jul 2017 12:09:07 +0800 Subject: [PATCH] [SPARK-21445] Make IntWrapper and LongWrapper in UTF8String Serializable ## What changes were proposed in this pull request? Making those two classes will avoid Serialization issues like below: ``` Caused by: java.io.NotSerializableException: org.apache.spark.unsafe.types.UTF8String$IntWrapper Serialization stack: - object not serializable (class: org.apache.spark.unsafe.types.UTF8String$IntWrapper, value: org.apache.spark.unsafe.types.UTF8String$IntWrapper326450e) - field (class: org.apache.spark.sql.catalyst.expressions.Cast$$anonfun$castToInt$1, name: result$2, type: class org.apache.spark.unsafe.types.UTF8String$IntWrapper) - object (class org.apache.spark.sql.catalyst.expressions.Cast$$anonfun$castToInt$1, ) ``` ## How was this patch tested? - [x] Manual testing - [ ] Unit test Author: Burak Yavuz Closes #18660 from brkyvz/serializableutf8. --- .../java/org/apache/spark/unsafe/types/UTF8String.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 9de4ca71ff..43f57672d9 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -863,8 +863,8 @@ public final class UTF8String implements Comparable, Externalizable, * Wrapper over `long` to allow result of parsing long from string to be accessed via reference. * This is done solely for better performance and is not expected to be used by end users. */ - public static class LongWrapper { - public long value = 0; + public static class LongWrapper implements Serializable { + public transient long value = 0; } /** @@ -874,8 +874,8 @@ public final class UTF8String implements Comparable, Externalizable, * {@link LongWrapper} could have been used here but using `int` directly save the extra cost of * conversion from `long` to `int` */ - public static class IntWrapper { - public int value = 0; + public static class IntWrapper implements Serializable { + public transient int value = 0; } /**