628932b8d0
Patch introduces the new way of working also retaining the existing ways of doing things.
For example build instruction for yarn in maven is
`mvn -Pyarn -PHadoop2.2 clean package -DskipTests`
in sbt it can become
`MAVEN_PROFILES="yarn, hadoop-2.2" sbt/sbt clean assembly`
Also supports
`sbt/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 clean assembly`
Author: Prashant Sharma <prashant.s@imaginea.com>
Author: Patrick Wendell <pwendell@gmail.com>
Closes #772 from ScrapCodes/sbt-maven and squashes the following commits:
a8ac951 [Prashant Sharma] Updated sbt version.
62b09bb [Prashant Sharma] Improvements.
fa6221d [Prashant Sharma] Excluding sql from mima
4b8875e [Prashant Sharma] Sbt assembly no longer builds tools by default.
72651ca [Prashant Sharma] Addresses code reivew comments.
acab73d [Prashant Sharma] Revert "Small fix to run-examples script."
ac4312c [Prashant Sharma] Revert "minor fix"
6af91ac [Prashant Sharma] Ported oldDeps back. + fixes issues with prev commit.
65cf06c [Prashant Sharma] Servelet API jars mess up with the other servlet jars on the class path.
446768e [Prashant Sharma] minor fix
89b9777 [Prashant Sharma] Merge conflicts
d0a02f2 [Prashant Sharma] Bumped up pom versions, Since the build now depends on pom it is better updated there. + general cleanups.
dccc8ac [Prashant Sharma] updated mima to check against 1.0
a49c61b [Prashant Sharma] Fix for tools jar
a2f5ae1 [Prashant Sharma] Fixes a bug in dependencies.
cf88758 [Prashant Sharma] cleanup
9439ea3 [Prashant Sharma] Small fix to run-examples script.
96cea1f [Prashant Sharma] SPARK-1776 Have Spark's SBT build read dependencies from Maven.
36efa62
[Patrick Wendell] Set project name in pom files and added eclipse/intellij plugins.
4973dbd [Patrick Wendell] Example build using pom reader.
103 lines
5.6 KiB
Scala
103 lines
5.6 KiB
Scala
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright ownership.
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
* (the "License"); you may not use this file except in compliance with
|
|
* the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
import com.typesafe.tools.mima.core._
|
|
|
|
/**
|
|
* Additional excludes for checking of Spark's binary compatibility.
|
|
*
|
|
* The Mima build will automatically exclude @DeveloperApi and @Experimental classes. This acts
|
|
* as an official audit of cases where we excluded other classes. Please use the narrowest
|
|
* possible exclude here. MIMA will usually tell you what exclude to use, e.g.:
|
|
*
|
|
* ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.rdd.RDD.take")
|
|
*
|
|
* It is also possible to exclude Spark classes and packages. This should be used sparingly:
|
|
*
|
|
* MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap")
|
|
*/
|
|
object MimaExcludes {
|
|
def excludes(version: String) =
|
|
version match {
|
|
case v if v.startsWith("1.1") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("deploy"),
|
|
MimaBuild.excludeSparkPackage("graphx")
|
|
) ++
|
|
Seq(
|
|
// Adding new method to JavaRDLike trait - we should probably mark this as a developer API.
|
|
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.api.java.JavaRDDLike.partitions"),
|
|
// We made a mistake earlier (ed06500d3) in the Java API to use default parameter values
|
|
// for countApproxDistinct* functions, which does not work in Java. We later removed
|
|
// them, and use the following to tell Mima to not care about them.
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
|
|
ProblemFilters.exclude[IncompatibleResultTypeProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaPairRDD.countApproxDistinctByKey$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaRDDLike.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.api.java.JavaDoubleRDD.countApproxDistinct$default$1"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.storage.MemoryStore.Entry"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.rdd.PairRDDFunctions.org$apache$spark$rdd$PairRDDFunctions$$"
|
|
+ "createZero$1")
|
|
) ++
|
|
Seq( // Ignore some private methods in ALS.
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$^dateFeatures"),
|
|
ProblemFilters.exclude[MissingMethodProblem]( // The only public constructor is the one without arguments.
|
|
"org.apache.spark.mllib.recommendation.ALS.this"),
|
|
ProblemFilters.exclude[MissingMethodProblem](
|
|
"org.apache.spark.mllib.recommendation.ALS.org$apache$spark$mllib$recommendation$ALS$$<init>$default$7")
|
|
) ++
|
|
MimaBuild.excludeSparkClass("rdd.ZippedRDD") ++
|
|
MimaBuild.excludeSparkClass("rdd.ZippedPartition") ++
|
|
MimaBuild.excludeSparkClass("util.SerializableHyperLogLog") ++
|
|
MimaBuild.excludeSparkClass("storage.Values") ++
|
|
MimaBuild.excludeSparkClass("storage.Entry") ++
|
|
MimaBuild.excludeSparkClass("storage.MemoryStore$Entry")
|
|
case v if v.startsWith("1.0") =>
|
|
Seq(
|
|
MimaBuild.excludeSparkPackage("api.java"),
|
|
MimaBuild.excludeSparkPackage("mllib"),
|
|
MimaBuild.excludeSparkPackage("streaming")
|
|
) ++
|
|
MimaBuild.excludeSparkClass("rdd.ClassTags") ++
|
|
MimaBuild.excludeSparkClass("util.XORShiftRandom") ++
|
|
MimaBuild.excludeSparkClass("graphx.EdgeRDD") ++
|
|
MimaBuild.excludeSparkClass("graphx.VertexRDD") ++
|
|
MimaBuild.excludeSparkClass("graphx.impl.GraphImpl") ++
|
|
MimaBuild.excludeSparkClass("graphx.impl.RoutingTable") ++
|
|
MimaBuild.excludeSparkClass("graphx.util.collection.PrimitiveKeyOpenHashMap") ++
|
|
MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap") ++
|
|
MimaBuild.excludeSparkClass("mllib.recommendation.MFDataGenerator") ++
|
|
MimaBuild.excludeSparkClass("mllib.optimization.SquaredGradient") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.RidgeRegressionWithSGD") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.LassoWithSGD") ++
|
|
MimaBuild.excludeSparkClass("mllib.regression.LinearRegressionWithSGD")
|
|
case _ => Seq()
|
|
}
|
|
}
|