[WIP] [SPARK-3996]: Shade Jetty in Spark deliverables
This patch piggy-back's on vanzin's work to simplify the Guava shading, and adds Jetty as a shaded library in Spark. Other than adding Jetty, it consilidates the \<artifactSet\>'s into the root pom. I found it was a bit easier to follow that way, since you don't need to look into child pom's to find out specific artifact sets included in shading. Author: Patrick Wendell <patrick@databricks.com> Closes #4252 from pwendell/jetty and squashes the following commits: 19f0710 [Patrick Wendell] More code review feedback 961452d [Patrick Wendell] Responding to feedback from Marcello 6df25ca [Patrick Wendell] [WIP] [SPARK-3996]: Shade Jetty in Spark deliverables
This commit is contained in:
parent
0bb15f22d1
commit
f240fe390b
|
@ -50,8 +50,8 @@ fi
|
|||
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
|
||||
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
|
||||
"classes ahead of assembly." >&2
|
||||
# Spark classes
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
|
@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then
|
|||
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
|
||||
# Jars for shaded deps in their original form (copied here during build)
|
||||
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
|
||||
fi
|
||||
|
||||
# Use spark-assembly jar from either RELEASE or assembly directory
|
||||
|
|
22
core/pom.xml
22
core/pom.xml
|
@ -94,22 +94,35 @@
|
|||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Jetty dependencies promoted to compile here so they are shaded
|
||||
and inlined into spark-core jar -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-plus</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-security</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-http</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
|
@ -348,19 +361,24 @@
|
|||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<executions>
|
||||
<!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
|
||||
shaded deps. So here we store jars in their original form which are added
|
||||
when the classpath is computed. -->
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<useSubDirectoryPerType>true</useSubDirectoryPerType>
|
||||
<includeArtifactIds>guava</includeArtifactIds>
|
||||
<includeArtifactIds>
|
||||
guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server
|
||||
</includeArtifactIds>
|
||||
<silent>true</silent>
|
||||
</configuration>
|
||||
</execution>
|
||||
|
|
|
@ -101,18 +101,6 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<configuration>
|
||||
<shadedArtifactAttached>false</shadedArtifactAttached>
|
||||
<artifactSet>
|
||||
<includes>
|
||||
<include>com.google.guava:guava</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
32
pom.xml
32
pom.xml
|
@ -337,25 +337,39 @@
|
|||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- Shaded deps marked as provided. These are promoted to compile scope
|
||||
in the modules where we want the shaded classes to appear in the
|
||||
associated jar. -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-http</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-security</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-plus</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<version>${jetty.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
|
@ -363,6 +377,8 @@
|
|||
<version>14.0.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- End of shaded deps -->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
|
@ -1276,10 +1292,26 @@
|
|||
<shadedArtifactAttached>false</shadedArtifactAttached>
|
||||
<artifactSet>
|
||||
<includes>
|
||||
<!-- At a minimum we must include this to force effective pom generation -->
|
||||
<include>org.spark-project.spark:unused</include>
|
||||
|
||||
<include>org.eclipse.jetty:jetty-io</include>
|
||||
<include>org.eclipse.jetty:jetty-http</include>
|
||||
<include>org.eclipse.jetty:jetty-plus</include>
|
||||
<include>org.eclipse.jetty:jetty-security</include>
|
||||
<include>org.eclipse.jetty:jetty-util</include>
|
||||
<include>org.eclipse.jetty:jetty-server</include>
|
||||
<include>com.google.guava:guava</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>org.eclipse.jetty</pattern>
|
||||
<shadedPattern>org.spark-project.jetty</shadedPattern>
|
||||
<includes>
|
||||
<include>org.eclipse.jetty.**</include>
|
||||
</includes>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.common</pattern>
|
||||
<shadedPattern>org.spark-project.guava</shadedPattern>
|
||||
|
|
Loading…
Reference in a new issue