ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-29202][DEPLOY] Driver java options are not passed to driver process in Yarn client mode

Browse source 
### What changes were proposed in this pull request?
`--driver-java-options` is not passed to driver process if the user runs the application in **Yarn client** mode

Run the below command

```
./bin/spark-sql --master yarn \
--driver-java-options="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5555"
```

**In Spark 2.4.4**
```
java ... -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5555
org.apache.spark.deploy.SparkSubmit --master yarn --conf spark.driver.extraJavaOptions=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5555 ...
```

**In Spark 3.0**
```
java ...
org.apache.spark.deploy.SparkSubmit --master yarn --conf spark.driver.extraJavaOptions=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5556 ...
```

This issue is caused by [SPARK-28980](https://github.com/apache/spark/pull/25684/files#diff-75e0f814aa3717db995fa701883dc4e1R395)

### Why are the changes needed?
Corrected the `isClientMode`  API implementation

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually,

![image](https://user-images.githubusercontent.com/35216143/65383114-c92dce80-dd2d-11e9-86c1-60e6d7e09f1e.png)

Closes #25889 from sandeep-katta/yarnmode.

Authored-by: sandeep katta <sandeep.katta2007@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
This commit is contained in:
sandeep katta 2019-09-26 15:37:20 -07:00 committed by Dongjoon Hyun
parent d3679a9782
commit 103de96059
2 changed files with 21 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
View file

@ -390,9 +390,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
String userMaster = firstNonEmpty(master, userProps.get(SparkLauncher.SPARK_MASTER));
String userDeployMode = firstNonEmpty(deployMode, userProps.get(SparkLauncher.DEPLOY_MODE));
// Default master is "local[*]", so assume client mode in that case
return userMaster == null ||
"client".equals(userDeployMode) ||
(!userMaster.equals("yarn") && userDeployMode == null);
return userMaster == null || userDeployMode == null || "client".equals(userDeployMode);
}
/**

20
launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
View file

@ -250,6 +250,26 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
new SparkSubmitCommandBuilder().buildSparkSubmitArgs();
}
@Test
public void testIsClientMode() {
// Default master is "local[*]"
SparkSubmitCommandBuilder builder = newCommandBuilder(Collections.emptyList());
assertTrue("By default application run in local mode",
builder.isClientMode(Collections.emptyMap()));
// --master yarn or it can be any RM
List<String> sparkSubmitArgs = Arrays.asList(parser.MASTER, "yarn");
builder = newCommandBuilder(sparkSubmitArgs);
assertTrue("By default deploy mode is client", builder.isClientMode(Collections.emptyMap()));
// --master yarn and set spark.submit.deployMode to client
Map<String, String> userProps = new HashMap<>();
userProps.put("spark.submit.deployMode", "client");
assertTrue(builder.isClientMode(userProps));
// --master mesos --deploy-mode cluster
sparkSubmitArgs = Arrays.asList(parser.MASTER, "mesos", parser.DEPLOY_MODE, "cluster");
builder = newCommandBuilder(sparkSubmitArgs);
assertFalse(builder.isClientMode(Collections.emptyMap()));
}
private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception {
final String DRIVER_DEFAULT_PARAM = "-Ddriver-default";
final String DRIVER_EXTRA_PARAM = "-Ddriver-extra";