ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-15665][CORE] spark-submit --kill and --status are not working

Browse source 
## What changes were proposed in this pull request?
--kill and --status were not considered while handling in OptionParser and due to that it was failing. Now handling the --kill and --status options as part of OptionParser.handle.

## How was this patch tested?
Added a test org.apache.spark.launcher.SparkSubmitCommandBuilderSuite.testCliKillAndStatus() and also I have verified these manually by running --kill and --status commands.

Author: Devaraj K <devaraj@apache.org>

Closes #13407 from devaraj-kavali/SPARK-15665.
This commit is contained in:
Devaraj K 2016-06-03 13:03:28 -07:00 committed by Marcelo Vanzin
parent 9e2eb13ca5
commit efd3b11a47
2 changed files with 29 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
View file

@ -89,7 +89,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
}
final List<String> sparkArgs;
private final boolean printInfo;
private final boolean isAppResourceReq;
private final boolean isExample;
/**
@ -101,7 +101,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
SparkSubmitCommandBuilder() {
this.sparkArgs = new ArrayList<>();
this.printInfo = false;
this.isAppResourceReq = true;
this.isExample = false;
}
@ -133,19 +133,19 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
this.isExample = isExample;
OptionParser parser = new OptionParser();
parser.parse(submitArgs);
this.printInfo = parser.infoRequested;
this.isAppResourceReq = parser.isAppResourceReq;
} else {
this.isExample = isExample;
this.printInfo = true;
this.isAppResourceReq = false;
}
}
@Override
public List<String> buildCommand(Map<String, String> env)
throws IOException, IllegalArgumentException {
if (PYSPARK_SHELL.equals(appResource) && !printInfo) {
if (PYSPARK_SHELL.equals(appResource) && isAppResourceReq) {
return buildPySparkShellCommand(env);
} else if (SPARKR_SHELL.equals(appResource) && !printInfo) {
} else if (SPARKR_SHELL.equals(appResource) && isAppResourceReq) {
return buildSparkRCommand(env);
} else {
return buildSparkSubmitCommand(env);
@ -156,7 +156,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
List<String> args = new ArrayList<>();
SparkSubmitOptionParser parser = new SparkSubmitOptionParser();
if (!allowsMixedArguments && !printInfo) {
if (!allowsMixedArguments && isAppResourceReq) {
checkArgument(appResource != null, "Missing application resource.");
}
@ -208,7 +208,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
args.add(join(",", pyFiles));
}
if (!printInfo) {
if (isAppResourceReq) {
checkArgument(!isExample || mainClass != null, "Missing example class name.");
}
if (mainClass != null) {
@ -388,7 +388,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
private class OptionParser extends SparkSubmitOptionParser {
boolean infoRequested = false;
boolean isAppResourceReq = true;
@Override
protected boolean handle(String opt, String value) {
@ -420,11 +420,15 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
allowsMixedArguments = true;
appResource = specialClasses.get(value);
}
} else if (opt.equals(KILL_SUBMISSION) || opt.equals(STATUS)) {
isAppResourceReq = false;
sparkArgs.add(opt);
sparkArgs.add(value);
} else if (opt.equals(HELP) || opt.equals(USAGE_ERROR)) {
infoRequested = true;
isAppResourceReq = false;
sparkArgs.add(opt);
} else if (opt.equals(VERSION)) {
infoRequested = true;
isAppResourceReq = false;
sparkArgs.add(opt);
} else {
sparkArgs.add(opt);

14
launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
View file

@ -72,6 +72,12 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
cmd.contains("org.apache.spark.deploy.SparkSubmit"));
}
@Test
public void testCliKillAndStatus() throws Exception {
testCLIOpts(parser.STATUS);
testCLIOpts(parser.KILL_SUBMISSION);
}
@Test
public void testCliParser() throws Exception {
List<String> sparkSubmitArgs = Arrays.asList(
@ -326,4 +332,12 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
return newCommandBuilder(args).buildCommand(env);
}
private void testCLIOpts(String opt) throws Exception {
List<String> helpArgs = Arrays.asList(opt, "driver-20160531171222-0000");
Map<String, String> env = new HashMap<>();
List<String> cmd = buildCommand(helpArgs, env);
assertTrue(opt + " should be contained in the final cmd.",
cmd.contains(opt));
}
}