[SPARK-27180][BUILD][YARN] Fix testing issues with yarn module in Hadoop-3
## What changes were proposed in this pull request? Fix testing issues with `yarn` module in Hadoop-3: 1. Upgrade jersey-1 to `1.19` to fix ```Cause: java.lang.NoClassDefFoundError: com/sun/jersey/spi/container/servlet/ServletContainer```. 2. Copy `ServerSocketUtil` from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java to fix ```java.lang.NoClassDefFoundError: org/apache/hadoop/net/ServerSocketUtil```. 3. Adapte `SessionHandler` from jetty-9.3.25.v20180904/jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java to fix ```java.lang.NoSuchMethodError: org.eclipse.jetty.server.session.SessionHandler.getSessionManager()Lorg/eclipse/jetty/server/SessionManager```. ## How was this patch tested? manual tests: ```shell build/sbt yarn/test -Pyarn build/sbt yarn/test -Phadoop-3.2 -Pyarn build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl resource-managers/yarn test -Pyarn build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl resource-managers/yarn test -Pyarn -Phadoop-3.2 ``` Closes #24115 from wangyum/hadoop3-yarn. Authored-by: Yuming Wang <yumwang@ebay.com> Signed-off-by: Sean Owen <sean.owen@databricks.com>
This commit is contained in:
Yuming Wang
Sean Owen
parent
57aff93886
commit
13c5c1fb4b
|
|
@ -115,3 +115,5 @@ structured-streaming/*
|
|||
kafka-source-initial-offset-version-2.1.0.bin
|
||||
kafka-source-initial-offset-future-version.bin
|
||||
vote.tmpl
|
||||
SessionManager.java
|
||||
SessionHandler.java
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
<name>Spark Project YARN</name>
|
||||
<properties>
|
||||
<sbt.project.name>yarn</sbt.project.name>
|
||||
<jersey-1.version>1.9</jersey-1.version>
|
||||
<jersey-1.version>1.19</jersey-1.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
|
@ -166,6 +166,12 @@
|
|||
<scope>test</scope>
|
||||
<version>${jersey-1.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<scope>test</scope>
|
||||
<version>${jersey-1.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- These dependencies are duplicated from core, because dependencies in the "provided"
|
||||
scope are not transitive.-->
|
||||
|
|
|
|||
|
|
@ -0,0 +1,132 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.net;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ServerSocket;
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* Copied from
|
||||
* hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
|
||||
* for Hadoop-3.x testing
|
||||
*/
|
||||
public class ServerSocketUtil {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ServerSocketUtil.class);
|
||||
private static Random rand = new Random();
|
||||
|
||||
/**
|
||||
* Port scan & allocate is how most other apps find ports
|
||||
*
|
||||
* @param port given port
|
||||
* @param retries number of retries
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
public static int getPort(int port, int retries) throws IOException {
|
||||
int tryPort = port;
|
||||
int tries = 0;
|
||||
while (true) {
|
||||
if (tries > 0 || tryPort == 0) {
|
||||
tryPort = port + rand.nextInt(65535 - port);
|
||||
}
|
||||
if (tryPort == 0) {
|
||||
continue;
|
||||
}
|
||||
try (ServerSocket s = new ServerSocket(tryPort)) {
|
||||
LOG.info("Using port " + tryPort);
|
||||
return tryPort;
|
||||
} catch (IOException e) {
|
||||
tries++;
|
||||
if (tries >= retries) {
|
||||
LOG.info("Port is already in use; giving up");
|
||||
throw e;
|
||||
} else {
|
||||
LOG.info("Port is already in use; trying again");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether port is available or not.
|
||||
*
|
||||
* @param port given port
|
||||
* @return
|
||||
*/
|
||||
private static boolean isPortAvailable(int port) {
|
||||
try (ServerSocket s = new ServerSocket(port)) {
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait till the port available.
|
||||
*
|
||||
* @param port given port
|
||||
* @param retries number of retries for given port
|
||||
* @return
|
||||
* @throws InterruptedException
|
||||
* @throws IOException
|
||||
*/
|
||||
public static int waitForPort(int port, int retries)
|
||||
throws InterruptedException, IOException {
|
||||
int tries = 0;
|
||||
while (true) {
|
||||
if (isPortAvailable(port)) {
|
||||
return port;
|
||||
} else {
|
||||
tries++;
|
||||
if (tries >= retries) {
|
||||
throw new IOException(
|
||||
"Port is already in use; giving up after " + tries + " times.");
|
||||
}
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the specified number of unique ports available.
|
||||
* The ports are all closed afterwards,
|
||||
* so other network services started may grab those same ports.
|
||||
*
|
||||
* @param numPorts number of required port nubmers
|
||||
* @return array of available port numbers
|
||||
* @throws IOException
|
||||
*/
|
||||
public static int[] getPorts(int numPorts) throws IOException {
|
||||
ServerSocket[] sockets = new ServerSocket[numPorts];
|
||||
int[] ports = new int[numPorts];
|
||||
for (int i = 0; i < numPorts; i++) {
|
||||
ServerSocket sock = new ServerSocket(0);
|
||||
sockets[i] = sock;
|
||||
ports[i] = sock.getLocalPort();
|
||||
}
|
||||
for (ServerSocket sock : sockets) {
|
||||
sock.close();
|
||||
}
|
||||
return ports;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
//
|
||||
// ========================================================================
|
||||
// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
|
||||
// ------------------------------------------------------------------------
|
||||
// All rights reserved. This program and the accompanying materials
|
||||
// are made available under the terms of the Eclipse Public License v1.0
|
||||
// and Apache License v2.0 which accompanies this distribution.
|
||||
//
|
||||
// The Eclipse Public License is available at
|
||||
// http://www.eclipse.org/legal/epl-v10.html
|
||||
//
|
||||
// The Apache License v2.0 is available at
|
||||
// http://www.opensource.org/licenses/apache2.0.php
|
||||
//
|
||||
// You may elect to redistribute this code under either of these licenses.
|
||||
// ========================================================================
|
||||
//
|
||||
|
||||
package org.eclipse.jetty.server;
|
||||
|
||||
import javax.servlet.SessionCookieConfig;
|
||||
import javax.servlet.SessionTrackingMode;
|
||||
import javax.servlet.http.Cookie;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpSession;
|
||||
import java.util.EventListener;
|
||||
import java.util.Set;
|
||||
|
||||
import org.eclipse.jetty.http.HttpCookie;
|
||||
import org.eclipse.jetty.server.session.SessionHandler;
|
||||
import org.eclipse.jetty.util.component.LifeCycle;
|
||||
|
||||
/**
|
||||
* Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
|
||||
* jetty-server/src/main/java/org/eclipse/jetty/server/SessionManager.java
|
||||
*/
|
||||
public interface SessionManager extends LifeCycle {
|
||||
/**
|
||||
* Session cookie name.
|
||||
* Defaults to <code>JSESSIONID</code>, but can be set with the
|
||||
* <code>org.eclipse.jetty.servlet.SessionCookie</code> context init parameter.
|
||||
*/
|
||||
String __SessionCookieProperty = "org.eclipse.jetty.servlet.SessionCookie";
|
||||
String __DefaultSessionCookie = "JSESSIONID";
|
||||
|
||||
/**
|
||||
* Session id path parameter name.
|
||||
* Defaults to <code>jsessionid</code>, but can be set with the
|
||||
* <code>org.eclipse.jetty.servlet.SessionIdPathParameterName</code> context init parameter.
|
||||
* If set to null or "none" no URL rewriting will be done.
|
||||
*/
|
||||
String __SessionIdPathParameterNameProperty =
|
||||
"org.eclipse.jetty.servlet.SessionIdPathParameterName";
|
||||
String __DefaultSessionIdPathParameterName = "jsessionid";
|
||||
String __CheckRemoteSessionEncoding = "org.eclipse.jetty.servlet.CheckingRemoteSessionIdEncoding";
|
||||
|
||||
/**
|
||||
* Session Domain.
|
||||
* If this property is set as a ServletContext InitParam, then it is
|
||||
* used as the domain for session cookies. If it is not set, then
|
||||
* no domain is specified for the session cookie.
|
||||
*/
|
||||
String __SessionDomainProperty = "org.eclipse.jetty.servlet.SessionDomain";
|
||||
String __DefaultSessionDomain = null;
|
||||
|
||||
/**
|
||||
* Session Path.
|
||||
* If this property is set as a ServletContext InitParam, then it is
|
||||
* used as the path for the session cookie. If it is not set, then
|
||||
* the context path is used as the path for the cookie.
|
||||
*/
|
||||
String __SessionPathProperty = "org.eclipse.jetty.servlet.SessionPath";
|
||||
|
||||
/**
|
||||
* Session Max Age.
|
||||
* If this property is set as a ServletContext InitParam, then it is
|
||||
* used as the max age for the session cookie. If it is not set, then
|
||||
* a max age of -1 is used.
|
||||
*/
|
||||
String __MaxAgeProperty = "org.eclipse.jetty.servlet.MaxAge";
|
||||
|
||||
/**
|
||||
* Returns the <code>HttpSession</code> with the given session id
|
||||
*
|
||||
* @param id the session id
|
||||
* @return the <code>HttpSession</code> with the corresponding id
|
||||
* or null if no session with the given id exists
|
||||
*/
|
||||
HttpSession getHttpSession(String id);
|
||||
|
||||
/**
|
||||
* Creates a new <code>HttpSession</code>.
|
||||
*
|
||||
* @param request the HttpServletRequest containing the requested session id
|
||||
* @return the new <code>HttpSession</code>
|
||||
*/
|
||||
HttpSession newHttpSession(HttpServletRequest request);
|
||||
|
||||
/**
|
||||
* @return true if session cookies should be HTTP-only (Microsoft extension)
|
||||
* @see HttpCookie#isHttpOnly()
|
||||
*/
|
||||
boolean getHttpOnly();
|
||||
|
||||
/**
|
||||
* @return the max period of inactivity, after which the session is invalidated, in seconds.
|
||||
* @see #setMaxInactiveInterval(int)
|
||||
*/
|
||||
int getMaxInactiveInterval();
|
||||
|
||||
/**
|
||||
* Sets the max period of inactivity, after which the session is invalidated, in seconds.
|
||||
*
|
||||
* @param seconds the max inactivity period, in seconds.
|
||||
* @see #getMaxInactiveInterval()
|
||||
*/
|
||||
void setMaxInactiveInterval(int seconds);
|
||||
|
||||
/**
|
||||
* Sets the {@link SessionHandler}.
|
||||
*
|
||||
* @param handler the <code>SessionHandler</code> object
|
||||
*/
|
||||
void setSessionHandler(SessionHandler handler);
|
||||
|
||||
/**
|
||||
* Adds an event listener for session-related events.
|
||||
*
|
||||
* @param listener the session event listener to add
|
||||
* Individual SessionManagers implementations may accept arbitrary listener types,
|
||||
* but they are expected to at least handle HttpSessionActivationListener,
|
||||
* HttpSessionAttributeListener,
|
||||
* HttpSessionBindingListener and HttpSessionListener.
|
||||
* @see #removeEventListener(EventListener)
|
||||
*/
|
||||
void addEventListener(EventListener listener);
|
||||
|
||||
/**
|
||||
* Removes an event listener for for session-related events.
|
||||
*
|
||||
* @param listener the session event listener to remove
|
||||
* @see #addEventListener(EventListener)
|
||||
*/
|
||||
void removeEventListener(EventListener listener);
|
||||
|
||||
/**
|
||||
* Removes all event listeners for session-related events.
|
||||
*
|
||||
* @see #removeEventListener(EventListener)
|
||||
*/
|
||||
void clearEventListeners();
|
||||
|
||||
/**
|
||||
* Gets a Cookie for a session.
|
||||
*
|
||||
* @param session the session to which the cookie should refer.
|
||||
* @param contextPath the context to which the cookie should be linked.
|
||||
* The client will only send the cookie value when
|
||||
* requesting resources under this path.
|
||||
* @param requestIsSecure whether the client is accessing the server over
|
||||
* a secure protocol (i.e. HTTPS).
|
||||
* @return if this <code>SessionManager</code> uses cookies, then this method will return a new
|
||||
* {@link Cookie cookie object} that should be set on the client
|
||||
* in order to link future HTTP requests
|
||||
* with the <code>session</code>. If cookies are not in use,
|
||||
* this method returns <code>null</code>.
|
||||
*/
|
||||
HttpCookie getSessionCookie(HttpSession session, String contextPath, boolean requestIsSecure);
|
||||
|
||||
/**
|
||||
* @return the cross context session id manager.
|
||||
* @see #setSessionIdManager(SessionIdManager)
|
||||
*/
|
||||
SessionIdManager getSessionIdManager();
|
||||
|
||||
/**
|
||||
* @return the cross context session id manager.
|
||||
* @deprecated use {@link #getSessionIdManager()}
|
||||
*/
|
||||
@Deprecated
|
||||
SessionIdManager getMetaManager();
|
||||
|
||||
/**
|
||||
* Sets the cross context session id manager
|
||||
*
|
||||
* @param idManager the cross context session id manager.
|
||||
* @see #getSessionIdManager()
|
||||
*/
|
||||
void setSessionIdManager(SessionIdManager idManager);
|
||||
|
||||
/**
|
||||
* @param session the session to test for validity
|
||||
* @return whether the given session is valid, that is, it has not been invalidated.
|
||||
*/
|
||||
boolean isValid(HttpSession session);
|
||||
|
||||
/**
|
||||
* @param session the session object
|
||||
* @return the unique id of the session within the cluster, extended with an optional node id.
|
||||
* @see #getClusterId(HttpSession)
|
||||
*/
|
||||
String getNodeId(HttpSession session);
|
||||
|
||||
/**
|
||||
* @param session the session object
|
||||
* @return the unique id of the session within the cluster (without a node id extension)
|
||||
* @see #getNodeId(HttpSession)
|
||||
*/
|
||||
String getClusterId(HttpSession session);
|
||||
|
||||
/**
|
||||
* Called by the {@link SessionHandler} when a session is first accessed by a request.
|
||||
*
|
||||
* @param session the session object
|
||||
* @param secure whether the request is secure or not
|
||||
* @return the session cookie. If not null,
|
||||
* this cookie should be set on the response to either migrate
|
||||
* the session or to refresh a session cookie that may expire.
|
||||
* @see #complete(HttpSession)
|
||||
*/
|
||||
HttpCookie access(HttpSession session, boolean secure);
|
||||
|
||||
/**
|
||||
* Called by the {@link SessionHandler} when a session is last accessed by a request.
|
||||
*
|
||||
* @param session the session object
|
||||
* @see #access(HttpSession, boolean)
|
||||
*/
|
||||
void complete(HttpSession session);
|
||||
|
||||
/**
|
||||
* Sets the session id URL path parameter name.
|
||||
*
|
||||
* @param parameterName the URL path parameter name
|
||||
* for session id URL rewriting (null or "none" for no rewriting).
|
||||
* @see #getSessionIdPathParameterName()
|
||||
* @see #getSessionIdPathParameterNamePrefix()
|
||||
*/
|
||||
void setSessionIdPathParameterName(String parameterName);
|
||||
|
||||
/**
|
||||
* @return the URL path parameter name for session id URL rewriting, by default "jsessionid".
|
||||
* @see #setSessionIdPathParameterName(String)
|
||||
*/
|
||||
String getSessionIdPathParameterName();
|
||||
|
||||
/**
|
||||
* @return a formatted version of {@link #getSessionIdPathParameterName()}, by default
|
||||
* ";" + sessionIdParameterName + "=", for easier lookup in URL strings.
|
||||
* @see #getSessionIdPathParameterName()
|
||||
*/
|
||||
String getSessionIdPathParameterNamePrefix();
|
||||
|
||||
/**
|
||||
* @return whether the session management is handled via cookies.
|
||||
*/
|
||||
boolean isUsingCookies();
|
||||
|
||||
/**
|
||||
* @return whether the session management is handled via URLs.
|
||||
*/
|
||||
boolean isUsingURLs();
|
||||
|
||||
Set<SessionTrackingMode> getDefaultSessionTrackingModes();
|
||||
|
||||
Set<SessionTrackingMode> getEffectiveSessionTrackingModes();
|
||||
|
||||
void setSessionTrackingModes(Set<SessionTrackingMode> sessionTrackingModes);
|
||||
|
||||
SessionCookieConfig getSessionCookieConfig();
|
||||
|
||||
/**
|
||||
* @return True if absolute URLs are check for remoteness before being session encoded.
|
||||
*/
|
||||
boolean isCheckingRemoteSessionIdEncoding();
|
||||
|
||||
/**
|
||||
* @param remote True if absolute URLs are check for remoteness before being session encoded.
|
||||
*/
|
||||
void setCheckingRemoteSessionIdEncoding(boolean remote);
|
||||
|
||||
/** Change the existing session id.
|
||||
*
|
||||
* @param oldClusterId the old cluster id
|
||||
* @param oldNodeId the old node id
|
||||
* @param newClusterId the new cluster id
|
||||
* @param newNodeId the new node id
|
||||
*/
|
||||
void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId);
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
//
|
||||
// ========================================================================
|
||||
// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
|
||||
// ------------------------------------------------------------------------
|
||||
// All rights reserved. This program and the accompanying materials
|
||||
// are made available under the terms of the Eclipse Public License v1.0
|
||||
// and Apache License v2.0 which accompanies this distribution.
|
||||
//
|
||||
// The Eclipse Public License is available at
|
||||
// http://www.eclipse.org/legal/epl-v10.html
|
||||
//
|
||||
// The Apache License v2.0 is available at
|
||||
// http://www.opensource.org/licenses/apache2.0.php
|
||||
//
|
||||
// You may elect to redistribute this code under either of these licenses.
|
||||
// ========================================================================
|
||||
//
|
||||
|
||||
package org.eclipse.jetty.server.session;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.eclipse.jetty.server.Request;
|
||||
import org.eclipse.jetty.server.SessionManager;
|
||||
import org.eclipse.jetty.server.handler.ScopedHandler;
|
||||
|
||||
/**
|
||||
* Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
|
||||
* jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java
|
||||
*/
|
||||
public class SessionHandler extends ScopedHandler {
|
||||
private SessionManager _sessionManager;
|
||||
|
||||
public SessionHandler() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param manager
|
||||
* The session manager
|
||||
*/
|
||||
public SessionHandler(SessionManager manager) {
|
||||
setSessionManager(manager);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the sessionManager.
|
||||
*/
|
||||
public SessionManager getSessionManager() {
|
||||
return _sessionManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param sessionManager
|
||||
* The sessionManager to set.
|
||||
*/
|
||||
public void setSessionManager(SessionManager sessionManager) {
|
||||
if (isStarted()) {
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
if (sessionManager != null) {
|
||||
updateBean(_sessionManager,sessionManager);
|
||||
_sessionManager=sessionManager;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* @see org.eclipse.jetty.server.Handler#handle(javax.servlet.http.HttpServletRequest,
|
||||
* javax.servlet.http.HttpServletResponse, int)
|
||||
*/
|
||||
@Override
|
||||
public void doHandle(String target, Request baseRequest, HttpServletRequest request,
|
||||
HttpServletResponse response) throws IOException, ServletException {
|
||||
// start manual inline of nextHandle(target,baseRequest,request,response);
|
||||
if (_nextScope != null && _nextScope == _handler) {
|
||||
_nextScope.doHandle(target,baseRequest,request,response);
|
||||
} else if (_handler != null) {
|
||||
_handler.handle(target,baseRequest,request,response);
|
||||
// end manual inline
|
||||
}
|
||||
}
|
||||
|
||||
public void clearEventListeners() {
|
||||
if (_sessionManager != null) {
|
||||
_sessionManager.clearEventListeners();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue