[SPARK-21671][CORE] Move kvstore to "util" sub-package, add private annotation.
Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #18886 from vanzin/SPARK-21671.
This commit is contained in:
parent
979bf946d5
commit
2c1bfb497f
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
|
@ -15,13 +15,15 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.lang.annotation.ElementType;
|
import java.lang.annotation.ElementType;
|
||||||
import java.lang.annotation.Retention;
|
import java.lang.annotation.Retention;
|
||||||
import java.lang.annotation.RetentionPolicy;
|
import java.lang.annotation.RetentionPolicy;
|
||||||
import java.lang.annotation.Target;
|
import java.lang.annotation.Target;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tags a field to be indexed when storing an object.
|
* Tags a field to be indexed when storing an object.
|
||||||
*
|
*
|
||||||
|
@ -46,6 +48,7 @@ import java.lang.annotation.Target;
|
||||||
* of those values.
|
* of those values.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
@Retention(RetentionPolicy.RUNTIME)
|
@Retention(RetentionPolicy.RUNTIME)
|
||||||
@Target({ElementType.FIELD, ElementType.METHOD})
|
@Target({ElementType.FIELD, ElementType.METHOD})
|
||||||
public @interface KVIndex {
|
public @interface KVIndex {
|
|
@ -15,10 +15,12 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abstraction for a local key/value store for storing app data.
|
* Abstraction for a local key/value store for storing app data.
|
||||||
*
|
*
|
||||||
|
@ -59,6 +61,7 @@ import java.io.Closeable;
|
||||||
* KVStore instances are thread-safe for both reads and writes.
|
* KVStore instances are thread-safe for both reads and writes.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public interface KVStore extends Closeable {
|
public interface KVStore extends Closeable {
|
||||||
|
|
||||||
/**
|
/**
|
|
@ -15,11 +15,13 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An iterator for KVStore.
|
* An iterator for KVStore.
|
||||||
*
|
*
|
||||||
|
@ -28,6 +30,7 @@ import java.util.List;
|
||||||
* explicitly close iterators after they're used.
|
* explicitly close iterators after they're used.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public interface KVStoreIterator<T> extends Iterator<T>, AutoCloseable {
|
public interface KVStoreIterator<T> extends Iterator<T>, AutoCloseable {
|
||||||
|
|
||||||
/**
|
/**
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
|
@ -25,6 +25,8 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Serializer used to translate between app-defined types and the LevelDB store.
|
* Serializer used to translate between app-defined types and the LevelDB store.
|
||||||
*
|
*
|
||||||
|
@ -33,6 +35,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
* and integers to be written as values directly, which will be written as UTF-8 strings.
|
* and integers to be written as values directly, which will be written as UTF-8 strings.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public class KVStoreSerializer {
|
public class KVStoreSerializer {
|
||||||
|
|
||||||
/**
|
/**
|
|
@ -15,10 +15,12 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A configurable view that allows iterating over values in a {@link KVStore}.
|
* A configurable view that allows iterating over values in a {@link KVStore}.
|
||||||
*
|
*
|
||||||
|
@ -33,6 +35,7 @@ import com.google.common.base.Preconditions;
|
||||||
* to be closed explicitly unless all elements are read.
|
* to be closed explicitly unless all elements are read.
|
||||||
* </p>
|
* </p>
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public abstract class KVStoreView<T> implements Iterable<T> {
|
public abstract class KVStoreView<T> implements Iterable<T> {
|
||||||
|
|
||||||
final Class<T> type;
|
final Class<T> type;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
import java.lang.reflect.Method;
|
import java.lang.reflect.Method;
|
||||||
|
@ -25,9 +25,12 @@ import java.util.stream.Stream;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around types managed in a KVStore, providing easy access to their indexed fields.
|
* Wrapper around types managed in a KVStore, providing easy access to their indexed fields.
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public class KVTypeInfo {
|
public class KVTypeInfo {
|
||||||
|
|
||||||
private final Class<?> type;
|
private final Class<?> type;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -36,9 +36,12 @@ import org.iq80.leveldb.DB;
|
||||||
import org.iq80.leveldb.Options;
|
import org.iq80.leveldb.Options;
|
||||||
import org.iq80.leveldb.WriteBatch;
|
import org.iq80.leveldb.WriteBatch;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implementation of KVStore that uses LevelDB as the underlying data store.
|
* Implementation of KVStore that uses LevelDB as the underlying data store.
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public class LevelDB implements KVStore {
|
public class LevelDB implements KVStore {
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.lang.reflect.Array;
|
import java.lang.reflect.Array;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
|
@ -15,13 +15,16 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.spark.annotation.Private;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Exception thrown when the store implementation is not compatible with the underlying data.
|
* Exception thrown when the store implementation is not compatible with the underlying data.
|
||||||
*/
|
*/
|
||||||
|
@Private
|
||||||
public class UnsupportedStoreVersionException extends IOException {
|
public class UnsupportedStoreVersionException extends IOException {
|
||||||
|
|
||||||
}
|
}
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.*;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
public class InMemoryIteratorSuite extends DBIteratorSuite {
|
public class InMemoryIteratorSuite extends DBIteratorSuite {
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.util.NoSuchElementException;
|
import java.util.NoSuchElementException;
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
|
@ -15,7 +15,7 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.spark.kvstore;
|
package org.apache.spark.util.kvstore;
|
||||||
|
|
||||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
|
Loading…
Reference in a new issue