ODIn/spark-instrumented-optimizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SPARK-15360][SPARK-SUBMIT] Should print spark-submit usage when no arguments is specified

Browse source 
(Please fill in changes proposed in this fix)
In 2.0, ./bin/spark-submit doesn't print out usage, but it raises an exception.
In this PR, an exception handling is added in the Main.java when the exception is thrown. In the handling code, if there is no additional argument, it prints out usage.

(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
Manually tested.
./bin/spark-submit
Usage: spark-submit [options] <app jar | python file> [app arguments]
Usage: spark-submit --kill [submission ID] --master [spark://...]
Usage: spark-submit --status [submission ID] --master [spark://...]
Usage: spark-submit run-example [options] example-class [example args]

Options:
  --master MASTER_URL         spark://host:port, mesos://host:port, yarn, or local.
  --deploy-mode DEPLOY_MODE   Whether to launch the driver program locally ("client") or
                              on one of the worker machines inside the cluster ("cluster")
                              (Default: client).
  --class CLASS_NAME          Your application's main class (for Java / Scala apps).
  --name NAME                 A name of your application.
  --jars JARS                 Comma-separated list of local jars to include on the driver
                              and executor classpaths.
  --packages                  Comma-separated list of maven coordinates of jars to include
                              on the driver and executor classpaths. Will search the local
                              maven repo, then maven central and any additional remote
                              repositories given by --repositories. The format for the
                              coordinates should be groupId:artifactId:version.

Author: wm624@hotmail.com <wm624@hotmail.com>

Closes #13163 from wangmiao1981/submit.
This commit is contained in:
wm624@hotmail.com 2016-05-20 10:27:41 -07:00 committed by Marcelo Vanzin
parent 2cbe96e64d
commit fe2fcb4803
2 changed files with 42 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
View file

@ -107,28 +107,37 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
SparkSubmitCommandBuilder(List<String> args) {
this.allowsMixedArguments = false;
this.sparkArgs = new ArrayList<>();
boolean isExample = false;
List<String> submitArgs = args;
if (args.size() > 0 && args.get(0).equals(PYSPARK_SHELL)) {
this.allowsMixedArguments = true;
appResource = PYSPARK_SHELL;
submitArgs = args.subList(1, args.size());
} else if (args.size() > 0 && args.get(0).equals(SPARKR_SHELL)) {
this.allowsMixedArguments = true;
appResource = SPARKR_SHELL;
submitArgs = args.subList(1, args.size());
} else if (args.size() > 0 && args.get(0).equals(RUN_EXAMPLE)) {
isExample = true;
submitArgs = args.subList(1, args.size());
if (args.size() > 0) {
switch (args.get(0)) {
case PYSPARK_SHELL:
this.allowsMixedArguments = true;
appResource = PYSPARK_SHELL;
submitArgs = args.subList(1, args.size());
break;
case SPARKR_SHELL:
this.allowsMixedArguments = true;
appResource = SPARKR_SHELL;
submitArgs = args.subList(1, args.size());
break;
case RUN_EXAMPLE:
isExample = true;
submitArgs = args.subList(1, args.size());
}
this.isExample = isExample;
OptionParser parser = new OptionParser();
parser.parse(submitArgs);
this.printInfo = parser.infoRequested;
} else {
this.isExample = isExample;
this.printInfo = true;
}
this.sparkArgs = new ArrayList<>();
this.isExample = isExample;
OptionParser parser = new OptionParser();
parser.parse(submitArgs);
this.printInfo = parser.infoRequested;
}
@Override
@ -147,7 +156,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
List<String> args = new ArrayList<>();
SparkSubmitOptionParser parser = new SparkSubmitOptionParser();
if (!allowsMixedArguments) {
if (!allowsMixedArguments && !printInfo) {
checkArgument(appResource != null, "Missing application resource.");
}

13
launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
View file

@ -58,6 +58,19 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
testCmdBuilder(false, false);
}
@Test
public void testCliHelpAndNoArg() throws Exception {
List<String> helpArgs = Arrays.asList(parser.HELP);
Map<String, String> env = new HashMap<>();
List<String> cmd = buildCommand(helpArgs, env);
assertTrue("--help should be contained in the final cmd.", cmd.contains(parser.HELP));
List<String> sparkEmptyArgs = Collections.emptyList();
cmd = buildCommand(sparkEmptyArgs, env);
assertTrue("org.apache.spark.deploy.SparkSubmit should be contained in the final cmd of empty input.",
cmd.contains("org.apache.spark.deploy.SparkSubmit"));
}
@Test
public void testCliParser() throws Exception {
List<String> sparkSubmitArgs = Arrays.asList(