b06389caec
https://issues.apache.org/jira/browse/SPARK-7805
Because `sql/hive`'s tests depend on the test jar of `sql/core`, we do not need to store `SQLTestUtils` and `ParquetTest` in `src/main`. We should only add stuff that will be needed by `sql/console` or Python tests (for Python, we need it in `src/main`, right? davies).
Author: Yin Huai <yhuai@databricks.com>
Closes #6334 from yhuai/SPARK-7805 and squashes the following commits:
af6d0c9 [Yin Huai] mima
b86746a [Yin Huai] Move SQLTestUtils.scala and ParquetTest.scala to src/test.
(cherry picked from commit ed21476bc0
)
Signed-off-by: Yin Huai <yhuai@databricks.com>
504 lines
31 KiB
Scala
504 lines
31 KiB
Scala
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership.
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
* (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
import com.typesafe.tools.mima.core._
|
|
import com.typesafe.tools.mima.core.ProblemFilters._
|
|
|
|
/**
|
|
* Additional excludes for checking of Spark's binary compatibility.
|
|
*
|
|
* The Mima build will automatically exclude @DeveloperApi and @Experimental classes. This acts
|
|
* as an official audit of cases where we excluded other classes. Please use the narrowest
|
|
* possible exclude here. MIMA will usually tell you what exclude to use, e.g.:
|
|
*
|
|
* ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.rdd.RDD.take")
|
|
*
|
|
* It is also possible to exclude Spark classes and packages. This should be used sparingly:
|
|
*
|
|
* MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap")
|
|
*/
|
|
object MimaExcludes {
|
|
def excludes(version: String) =
|
|
version match {
|
|
case v if v.startsWith("1.4") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("deploy"),
|
|
MimaBuild.excludeSparkPackage("ml"),
|
|
// SPARK-5922 Adding a generalized diff(other: RDD[(VertexId, VD)]) to VertexRDD
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.graphx.VertexRDD.diff"),
|
|
// These are needed if checking against the sbt build, since they are part of
|
|
// the maven-generated artifacts in 1.3.
|
|
excludePackage("org.spark-project.jetty"),
|
|
MimaBuild.excludeSparkPackage("unused"),
|
|
ProblemFilters.exclude[MissingClassProblem]("com.google.common.base.Optional"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.rdd.JdbcRDD.compute"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.broadcast.HttpBroadcastFactory.newBroadcast"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.scheduler.OutputCommitCoordinator$OutputCommitCoordinatorActor")
|
|
) ++ Seq(
|
|
// SPARK-4655 - Making Stage an Abstract class broke binary compatility even though
|
|
// the stage class is defined as private[spark]
|
|
ProblemFilters.exclude[AbstractClassProblem]("org.apache.spark.scheduler.Stage")
|
|
) ++ Seq(
|
|
// SPARK-6510 Add a Graph#minus method acting as Set#difference
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.graphx.VertexRDD.minus")
|
|
) ++ Seq(
|
|
// SPARK-6492 Fix deadlock in SparkContext.stop()
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.SparkContext.org$" +
|
|
"apache$spark$SparkContext$$SPARK_CONTEXT_CONSTRUCTOR_LOCK")
|
|
)++ Seq(
|
|
// SPARK-6693 add tostring with max lines and width for matrix
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrix.toString")
|
|
)++ Seq(
|
|
// SPARK-6703 Add getOrCreate method to SparkContext
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem]
|
|
("org.apache.spark.SparkContext.org$apache$spark$SparkContext$$activeContext")
|
|
)++ Seq(
|
|
// SPARK-7090 Introduce LDAOptimizer to LDA to further improve extensibility
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.mllib.clustering.LDA$EMOptimizer")
|
|
) ++ Seq(
|
|
// SPARK-6756 add toSparse, toDense, numActives, numNonzeros, and compressed to Vector
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Vector.compressed"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Vector.toDense"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Vector.numNonzeros"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Vector.toSparse"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Vector.numActives"),
|
|
// SPARK-7681 add SparseVector support for gemv
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrix.multiply"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.DenseMatrix.multiply"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.SparseMatrix.multiply")
|
|
) ++ Seq(
|
|
// Execution should never be included as its always internal.
|
|
MimaBuild.excludeSparkPackage("sql.execution"),
|
|
// This `protected[sql]` method was removed in 1.3.1
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.sql.SQLContext.checkAnalysis"),
|
|
// These `private[sql]` class were removed in 1.4.0:
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.execution.AddExchange"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.execution.AddExchange$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.PartitionSpec"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.PartitionSpec$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.Partition"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.Partition$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetRelation2$PartitionValues"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetRelation2$PartitionValues$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetRelation2"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetRelation2$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetRelation2$MetadataCache"),
|
|
// These test support classes were moved out of src/main and into src/test:
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetTestData"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetTestData$"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.TestGroupWriteSupport"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.CachedData"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.CachedData$"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.CacheManager"),
|
|
// TODO: Remove the following rule once ParquetTest has been moved to src/test.
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.sql.parquet.ParquetTest")
|
|
) ++ Seq(
|
|
// SPARK-7530 Added StreamingContext.getState()
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.StreamingContext.state_=")
|
|
) ++ Seq(
|
|
// SPARK-7081 changed ShuffleWriter from a trait to an abstract class and removed some
|
|
// unnecessary type bounds in order to fix some compiler warnings that occurred when
|
|
// implementing this interface in Java. Note that ShuffleWriter is private[spark].
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.shuffle.ShuffleWriter")
|
|
) ++ Seq(
|
|
// SPARK-6888 make jdbc driver handling user definable
|
|
// This patch renames some classes to API friendly names.
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.jdbc.DriverQuirks$"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.jdbc.DriverQuirks"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.jdbc.PostgresQuirks"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.jdbc.NoQuirks"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.jdbc.MySQLQuirks")
|
|
)
|
|
|
|
case v if v.startsWith("1.3") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("deploy"),
|
|
MimaBuild.excludeSparkPackage("ml"),
|
|
// These are needed if checking against the sbt build, since they are part of
|
|
// the maven-generated artifacts in the 1.2 build.
|
|
MimaBuild.excludeSparkPackage("unused"),
|
|
ProblemFilters.exclude[MissingClassProblem]("com.google.common.base.Optional")
|
|
) ++ Seq(
|
|
// SPARK-2321
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.SparkStageInfoImpl.this"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.SparkStageInfo.submissionTime")
|
|
) ++ Seq(
|
|
// SPARK-4614
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrices.randn"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrices.rand")
|
|
) ++ Seq(
|
|
// SPARK-5321
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.SparseMatrix.transposeMultiply"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrix.transpose"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.DenseMatrix.transposeMultiply"),
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.linalg.Matrix." +
|
|
"org$apache$spark$mllib$linalg$Matrix$_setter_$isTransposed_="),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrix.isTransposed"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.linalg.Matrix.foreachActive")
|
|
) ++ Seq(
|
|
// SPARK-5540
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.solveLeastSquares"),
|
|
// SPARK-5536
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateBlock")
|
|
) ++ Seq(
|
|
// SPARK-3325
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.api.java.JavaDStreamLike.print"),
|
|
// SPARK-2757
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.streaming.flume.sink.SparkAvroCallbackHandler." +
|
|
"removeAndGetProcessor")
|
|
) ++ Seq(
|
|
// SPARK-5123 (SparkSQL data type change) - alpha component only
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.ml.feature.HashingTF.outputDataType"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.ml.feature.Tokenizer.outputDataType"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.ml.feature.Tokenizer.validateInputType"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.ml.classification.LogisticRegressionModel.validateAndTransformSchema"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.ml.classification.LogisticRegression.validateAndTransformSchema")
|
|
) ++ Seq(
|
|
// SPARK-4014
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.TaskContext.taskAttemptId"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.TaskContext.attemptNumber")
|
|
) ++ Seq(
|
|
// SPARK-5166 Spark SQL API stabilization
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.Transformer.transform"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.Estimator.fit"),
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.ml.Transformer.transform"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.Pipeline.fit"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.PipelineModel.transform"),
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.ml.Estimator.fit"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.Evaluator.evaluate"),
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.ml.Evaluator.evaluate"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.tuning.CrossValidator.fit"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.tuning.CrossValidatorModel.transform"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.feature.StandardScaler.fit"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.feature.StandardScalerModel.transform"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.classification.LogisticRegressionModel.transform"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.classification.LogisticRegression.fit"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.evaluation.BinaryClassificationEvaluator.evaluate")
|
|
) ++ Seq(
|
|
// SPARK-5270
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.isEmpty")
|
|
) ++ Seq(
|
|
// SPARK-5430
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.treeReduce"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.treeAggregate")
|
|
) ++ Seq(
|
|
// SPARK-5297 Java FileStream do not work with custom key/values
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.api.java.JavaStreamingContext.fileStream")
|
|
) ++ Seq(
|
|
// SPARK-5315 Spark Streaming Java API returns Scala DStream
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.api.java.JavaDStreamLike.reduceByWindow")
|
|
) ++ Seq(
|
|
// SPARK-5461 Graph should have isCheckpointed, getCheckpointFiles methods
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.graphx.Graph.getCheckpointFiles"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.graphx.Graph.isCheckpointed")
|
|
) ++ Seq(
|
|
// SPARK-4789 Standardize ML Prediction APIs
|
|
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.mllib.linalg.VectorUDT"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.mllib.linalg.VectorUDT.serialize"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.mllib.linalg.VectorUDT.sqlType")
|
|
) ++ Seq(
|
|
// SPARK-5814
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$wrapDoubleArray"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$fillFullMatrix"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$iterations"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$makeOutLinkBlock"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$computeYtY"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$makeLinkRDDs"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$alpha"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$randomFactor"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$makeInLinkBlock"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$dspr"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$lambda"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$implicitPrefs"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$rank")
|
|
) ++ Seq(
|
|
// SPARK-4682
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.RealClock"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.Clock"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.TestClock")
|
|
) ++ Seq(
|
|
// SPARK-5922 Adding a generalized diff(other: RDD[(VertexId, VD)]) to VertexRDD
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.graphx.VertexRDD.diff")
|
|
)
|
|
|
|
case v if v.startsWith("1.2") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("deploy"),
|
|
MimaBuild.excludeSparkPackage("graphx")
|
|
) ++
|
|
MimaBuild.excludeSparkClass("mllib.linalg.Matrix") ++
|
|
MimaBuild.excludeSparkClass("mllib.linalg.Vector") ++
|
|
Seq(
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.scheduler.TaskLocation"),
|
|
// Added normL1 and normL2 to trait MultivariateStatisticalSummary
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.stat.MultivariateStatisticalSummary.normL1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.stat.MultivariateStatisticalSummary.normL2"),
|
|
// MapStatus should be private[spark]
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.scheduler.MapStatus"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.network.netty.PathResolver"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.spark.network.netty.client.BlockClientListener"),
|
|
|
|
// TaskContext was promoted to Abstract class
|
|
ProblemFilters.exclude[AbstractClassProblem](
|
|
"org.apache.spark.TaskContext"),
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.util.collection.SortDataFormat")
|
|
) ++ Seq(
|
|
// Adding new methods to the JavaRDDLike trait:
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.takeAsync"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.foreachPartitionAsync"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.countAsync"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.foreachAsync"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.collectAsync")
|
|
) ++ Seq(
|
|
// SPARK-3822
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.SparkContext.org$apache$spark$SparkContext$$createTaskScheduler")
|
|
) ++ Seq(
|
|
// SPARK-1209
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.hadoop.mapreduce.SparkHadoopMapReduceUtil"),
|
|
ProblemFilters.exclude[MissingClassProblem](
|
|
"org.apache.hadoop.mapred.SparkHadoopMapRedUtil"),
|
|
ProblemFilters.exclude[MissingTypesProblem](
|
|
"org.apache.spark.rdd.PairRDDFunctions")
|
|
) ++ Seq(
|
|
// SPARK-4062
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.kafka.KafkaReceiver#MessageHandler.this")
|
|
)
|
|
|
|
case v if v.startsWith("1.1") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("deploy"),
|
|
MimaBuild.excludeSparkPackage("graphx")
|
|
) ++
|
|
Seq(
|
|
// Adding new method to JavaRDLike trait - we should probably mark this as a developer API.
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.api.java.JavaRDDLike.partitions"),
|
|
// Should probably mark this as Experimental
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.foreachAsync"),
|
|
// We made a mistake earlier (ed06500d3) in the Java API to use default parameter values
|
|
// for countApproxDistinct* functions, which does not work in Java. We later removed
|
|
// them, and use the following to tell Mima to not care about them.
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaDoubleRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.DiskStore.getValues"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.MemoryStore.Entry")
|
|
) ++
|
|
Seq(
|
|
// Serializer interface change. See SPARK-3045.
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.serializer.DeserializationStream"),
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.serializer.Serializer"),
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.serializer.SerializationStream"),
|
|
ProblemFilters.exclude[IncompatibleTemplateDefProblem](
|
|
"org.apache.spark.serializer.SerializerInstance")
|
|
)++
|
|
Seq(
|
|
// Renamed putValues -> putArray + putIterator
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.MemoryStore.putValues"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.DiskStore.putValues"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.TachyonStore.putValues")
|
|
) ++
|
|
Seq(
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.streaming.flume.FlumeReceiver.this"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.streaming.kafka.KafkaUtils.createStream"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.streaming.kafka.KafkaReceiver.this")
|
|
) ++
|
|
Seq( // Ignore some private methods in ALS.
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures"),
|
|
ProblemFilters.exclude[MissingMethodProblem]( // The only public constructor is the one without arguments.
|
|
"org.apache.spark.mllib.recommendation.ALS.this"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$<init>$default$7"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures")
|
|
) ++
|
|
MimaBuild.excludeSparkClass("mllib.linalg.distributed.ColumnStatisticsAggregator") ++
|
|
MimaBuild.excludeSparkClass("rdd.ZippedRDD") ++
|
|
MimaBuild.excludeSparkClass("rdd.ZippedPartition") ++
|
|
MimaBuild.excludeSparkClass("util.SerializableHyperLogLog") ++
|
|
MimaBuild.excludeSparkClass("storage.Values") ++
|
|
MimaBuild.excludeSparkClass("storage.Entry") ++
|
|
MimaBuild.excludeSparkClass("storage.MemoryStore$Entry") ++
|
|
// Class was missing "@DeveloperApi" annotation in 1.0.
|
|
MimaBuild.excludeSparkClass("scheduler.SparkListenerApplicationStart") ++
|
|
Seq(
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.mllib.tree.impurity.Gini.calculate"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.mllib.tree.impurity.Entropy.calculate"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem](
|
|
"org.apache.spark.mllib.tree.impurity.Variance.calculate")
|
|
) ++
|
|
Seq( // Package-private classes removed in SPARK-2341
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.BinaryLabelParser"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.BinaryLabelParser$"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.LabelParser"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.LabelParser$"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.MulticlassLabelParser"),
|
|
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.util.MulticlassLabelParser$")
|
|
) ++
|
|
Seq( // package-private classes removed in MLlib
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.regression.GeneralizedLinearAlgorithm.org$apache$spark$mllib$regression$GeneralizedLinearAlgorithm$$prependOne")
|
|
) ++
|
|
Seq( // new Vector methods in MLlib (binary compatible assuming users do not implement Vector)
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.linalg.Vector.copy")
|
|
) ++
|
|
Seq( // synthetic methods generated in LabeledPoint
|
|
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.mllib.regression.LabeledPoint$"),
|
|
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.mllib.regression.LabeledPoint.apply"),
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.regression.LabeledPoint.toString")
|
|
) ++
|
|
Seq ( // Scala 2.11 compatibility fix
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.StreamingContext.<init>$default$2")
|
|
)
|
|
case v if v.startsWith("1.0") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("api.java"),
|
|
MimaBuild.excludeSparkPackage("mllib"),
|
|
MimaBuild.excludeSparkPackage("streaming")
|
|
) ++
|
|
MimaBuild.excludeSparkClass("rdd.ClassTags") ++
|
|
MimaBuild.excludeSparkClass("util.XORShiftRandom") ++
|
|
MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++
|
|
MimaBuild.excludeSparkClass("graphx.VertexRDD") ++
|
|
MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++
|
|
MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++
|
|
MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++
|
|
MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++
|
|
MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
|
|
MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
|
|
case _ => Seq()
|
|
}
|
|
}
|