[SPARK-22269][BUILD] Run Java linter via SBT for Jenkins
## What changes were proposed in this pull request? This PR proposes to check Java lint via SBT for Jenkins. It uses the SBT wrapper for checkstyle. I manually tested. If we build the codes once, running this script takes 2 mins at maximum in my local: Test codes: ``` Checkstyle failed at following occurrences: [error] Checkstyle error found in /.../spark/core/src/test/java/test/org/apache/spark/JavaAPISuite.java:82: Line is longer than 100 characters (found 103). [error] 1 issue(s) found in Checkstyle report: /.../spark/core/target/checkstyle-test-report.xml [error] Checkstyle error found in /.../spark/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaDataFrameSuite.java:84: Line is longer than 100 characters (found 115). [error] 1 issue(s) found in Checkstyle report: /.../spark/sql/hive/target/checkstyle-test-report.xml ... ``` Main codes: ``` Checkstyle failed at following occurrences: [error] Checkstyle error found in /.../spark/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartition.java:39: Line is longer than 100 characters (found 104). [error] Checkstyle error found in /.../spark/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartitionReader.java:26: Line is longer than 100 characters (found 110). [error] Checkstyle error found in /.../spark/sql/core/src/main/java/org/apache/spark/sql/sources/v2/reader/InputPartitionReader.java:30: Line is longer than 100 characters (found 104). ... ``` ## How was this patch tested? Manually tested. Jenkins build should test this. Author: hyukjinkwon <gurwls223@apache.org> Closes #21399 from HyukjinKwon/SPARK-22269.
This commit is contained in:
hyukjinkwon
parent
8a545822d0
commit
4a14dc0aff
|
|
@ -204,7 +204,7 @@ def run_scala_style_checks():
|
|||
|
||||
def run_java_style_checks():
|
||||
set_title_and_block("Running Java style checks", "BLOCK_JAVA_STYLE")
|
||||
run_cmd([os.path.join(SPARK_HOME, "dev", "lint-java")])
|
||||
run_cmd([os.path.join(SPARK_HOME, "dev", "sbt-checkstyle")])
|
||||
|
||||
|
||||
def run_python_style_checks():
|
||||
|
|
@ -574,8 +574,7 @@ def main():
|
|||
or f.endswith("checkstyle.xml")
|
||||
or f.endswith("checkstyle-suppressions.xml")
|
||||
for f in changed_files):
|
||||
# run_java_style_checks()
|
||||
pass
|
||||
run_java_style_checks()
|
||||
if not changed_files or any(f.endswith("lint-python")
|
||||
or f.endswith("tox.ini")
|
||||
or f.endswith(".py")
|
||||
|
|
|
|||
42
dev/sbt-checkstyle
Executable file
42
dev/sbt-checkstyle
Executable file
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file
|
||||
# with failure (either resolution or compilation); the "q" makes SBT quit.
|
||||
ERRORS=$(echo -e "q\n" \
|
||||
| build/sbt \
|
||||
-Pkinesis-asl \
|
||||
-Pmesos \
|
||||
-Pkafka-0-8 \
|
||||
-Pkubernetes \
|
||||
-Pyarn \
|
||||
-Pflume \
|
||||
-Phive \
|
||||
-Phive-thriftserver \
|
||||
checkstyle test:checkstyle \
|
||||
| awk '{if($1~/error/)print}' \
|
||||
)
|
||||
|
||||
if test ! -z "$ERRORS"; then
|
||||
echo -e "Checkstyle failed at following occurrences:\n$ERRORS"
|
||||
exit 1
|
||||
else
|
||||
echo -e "Checkstyle checks passed."
|
||||
fi
|
||||
|
||||
|
|
@ -27,6 +27,7 @@ import sbt._
|
|||
import sbt.Classpaths.publishTask
|
||||
import sbt.Keys._
|
||||
import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
|
||||
import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport._
|
||||
import com.simplytyped.Antlr4Plugin._
|
||||
import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
|
||||
import com.typesafe.tools.mima.plugin.MimaKeys
|
||||
|
|
@ -317,7 +318,7 @@ object SparkBuild extends PomBuild {
|
|||
/* Enable shared settings on all projects */
|
||||
(allProjects ++ optionallyEnabledProjects ++ assemblyProjects ++ copyJarsProjects ++ Seq(spark, tools))
|
||||
.foreach(enable(sharedSettings ++ DependencyOverrides.settings ++
|
||||
ExcludedDependencies.settings))
|
||||
ExcludedDependencies.settings ++ Checkstyle.settings))
|
||||
|
||||
/* Enable tests settings for all projects except examples, assembly and tools */
|
||||
(allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings))
|
||||
|
|
@ -740,6 +741,17 @@ object Unidoc {
|
|||
)
|
||||
}
|
||||
|
||||
object Checkstyle {
|
||||
lazy val settings = Seq(
|
||||
checkstyleSeverityLevel := Some(CheckstyleSeverityLevel.Error),
|
||||
javaSource in (Compile, checkstyle) := baseDirectory.value / "src/main/java",
|
||||
javaSource in (Test, checkstyle) := baseDirectory.value / "src/test/java",
|
||||
checkstyleConfigLocation := CheckstyleConfigLocation.File("dev/checkstyle.xml"),
|
||||
checkstyleOutputFile := baseDirectory.value / "target/checkstyle-output.xml",
|
||||
checkstyleOutputFile in Test := baseDirectory.value / "target/checkstyle-output.xml"
|
||||
)
|
||||
}
|
||||
|
||||
object CopyDependencies {
|
||||
|
||||
val copyDeps = TaskKey[Unit]("copyDeps", "Copies needed dependencies to the build directory.")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,11 @@
|
|||
addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "3.1.1")
|
||||
|
||||
// sbt-checkstyle-plugin uses an old version of checkstyle. Match it to Maven's.
|
||||
libraryDependencies += "com.puppycrawl.tools" % "checkstyle" % "8.2"
|
||||
|
||||
// checkstyle uses guava 23.0.
|
||||
libraryDependencies += "com.google.guava" % "guava" % "23.0"
|
||||
|
||||
// need to make changes to uptake sbt 1.0 support in "com.eed3si9n" % "sbt-assembly" % "1.14.5"
|
||||
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue