[SPARK-14134][CORE] Change the package name used for shading classes.
The current package name uses a dash, which is a little weird but seemed to work. That is, until a new test tried to mock a class that references one of those shaded types, and then things started failing. Most changes are just noise to fix the logging configs. For reference, SPARK-8815 also raised this issue, although at the time it did not cause any issues in Spark, so it was not addressed. Author: Marcelo Vanzin <vanzin@cloudera.com> Closes #11941 from vanzin/SPARK-14134.
This commit is contained in:
parent
d76592276f
commit
21d5ca128b
|
@ -36,7 +36,7 @@
|
|||
<!-- Make sure all Hadoop dependencies are provided to avoid repackaging. -->
|
||||
<hadoop.deps.scope>provided</hadoop.deps.scope>
|
||||
<shuffle.jar>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</shuffle.jar>
|
||||
<shade>org/spark-project/</shade>
|
||||
<shade>org/spark_project/</shade>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -91,7 +91,7 @@
|
|||
<relocations>
|
||||
<relocation>
|
||||
<pattern>com.fasterxml.jackson</pattern>
|
||||
<shadedPattern>org.spark-project.com.fasterxml.jackson</shadedPattern>
|
||||
<shadedPattern>${spark.shade.packageName}.com.fasterxml.jackson</shadedPattern>
|
||||
<includes>
|
||||
<include>com.fasterxml.jackson.**</include>
|
||||
</includes>
|
||||
|
|
|
@ -28,8 +28,8 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}:
|
|||
log4j.logger.org.apache.spark.repl.Main=WARN
|
||||
|
||||
# Settings to quiet third party logs that are too verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
|
||||
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
|
||||
log4j.logger.org.apache.parquet=ERROR
|
||||
|
|
|
@ -28,8 +28,8 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}:
|
|||
log4j.logger.org.apache.spark.repl.Main=WARN
|
||||
|
||||
# Settings to quiet third party logs that are too verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
|
||||
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
|
||||
|
||||
|
|
|
@ -33,5 +33,4 @@ log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.console.layout.ConversionPattern=%t: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
org.spark-project.jetty.LEVEL=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,5 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
||||
|
|
|
@ -24,5 +24,5 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
||||
|
|
|
@ -24,4 +24,4 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,5 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
||||
|
|
|
@ -31,7 +31,7 @@ log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
|
||||
|
||||
# Settings to quiet third party logs that are too verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
|
||||
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
|
||||
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
|
|
@ -24,4 +24,4 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,4 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
org.spark-project.jetty.LEVEL=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -30,5 +30,4 @@ log4j.appender.childproc.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.childproc.layout.ConversionPattern=%t: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
org.spark-project.jetty.LEVEL=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,5 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
||||
|
|
7
pom.xml
7
pom.xml
|
@ -182,6 +182,9 @@
|
|||
<test.java.home>${java.home}</test.java.home>
|
||||
<test.exclude.tags></test.exclude.tags>
|
||||
|
||||
<!-- Package to use when relocating shaded classes. -->
|
||||
<spark.shade.packageName>org.spark_project</spark.shade.packageName>
|
||||
|
||||
<!-- Modules that copy jars to the build directory should do so under this location. -->
|
||||
<jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
|
||||
|
||||
|
@ -2204,14 +2207,14 @@
|
|||
<relocations>
|
||||
<relocation>
|
||||
<pattern>org.eclipse.jetty</pattern>
|
||||
<shadedPattern>org.spark-project.jetty</shadedPattern>
|
||||
<shadedPattern>${spark.shade.packageName}.jetty</shadedPattern>
|
||||
<includes>
|
||||
<include>org.eclipse.jetty.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.common</pattern>
|
||||
<shadedPattern>org.spark-project.guava</shadedPattern>
|
||||
<shadedPattern>${spark.shade.packageName}.guava</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
</configuration>
|
||||
|
|
|
@ -24,4 +24,4 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,4 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
org.spark-project.jetty.LEVEL=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -24,5 +24,5 @@ log4j.appender.file.layout=org.apache.log4j.PatternLayout
|
|||
log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n
|
||||
|
||||
# Ignore messages below warning level from Jetty, because it's a bit verbose
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
||||
|
|
|
@ -28,4 +28,4 @@ log4j.logger.com.sun.jersey=WARN
|
|||
log4j.logger.org.apache.hadoop=WARN
|
||||
log4j.logger.org.eclipse.jetty=WARN
|
||||
log4j.logger.org.mortbay=WARN
|
||||
log4j.logger.org.spark-project.jetty=WARN
|
||||
log4j.logger.org.spark_project.jetty=WARN
|
||||
|
|
|
@ -53,7 +53,7 @@ abstract class BaseYarnClusterSuite
|
|||
|log4j.logger.org.apache.hadoop=WARN
|
||||
|log4j.logger.org.eclipse.jetty=WARN
|
||||
|log4j.logger.org.mortbay=WARN
|
||||
|log4j.logger.org.spark-project.jetty=WARN
|
||||
|log4j.logger.org.spark_project.jetty=WARN
|
||||
""".stripMargin
|
||||
|
||||
private var yarnCluster: MiniYARNCluster = _
|
||||
|
|
Loading…
Reference in a new issue