91f9504e60
Author: Chris Fregly <chris@fregly.com> Closes #1434 from cfregly/master and squashes the following commits: 4774581 [Chris Fregly] updated docs, renamed retry to retryRandom to be more clear, removed retries around store() method 0393795 [Chris Fregly] moved Kinesis examples out of examples/ and back into extras/kinesis-asl 691a6be [Chris Fregly] fixed tests and formatting, fixed a bug with JavaKinesisWordCount during union of streams 0e1c67b [Chris Fregly] Merge remote-tracking branch 'upstream/master' 74e5c7c [Chris Fregly] updated per TD's feedback. simplified examples, updated docs e33cbeb [Chris Fregly] Merge remote-tracking branch 'upstream/master' bf614e9 [Chris Fregly] per matei's feedback: moved the kinesis examples into the examples/ dir d17ca6d [Chris Fregly] per TD's feedback: updated docs, simplified the KinesisUtils api 912640c [Chris Fregly] changed the foundKinesis class to be a publically-avail class db3eefd [Chris Fregly] Merge remote-tracking branch 'upstream/master' 21de67f [Chris Fregly] Merge remote-tracking branch 'upstream/master' 6c39561 [Chris Fregly] parameterized the versions of the aws java sdk and kinesis client 338997e [Chris Fregly] improve build docs for kinesis 828f8ae [Chris Fregly] more cleanup e7c8978 [Chris Fregly] Merge remote-tracking branch 'upstream/master' cd68c0d [Chris Fregly] fixed typos and backward compatibility d18e680 [Chris Fregly] Merge remote-tracking branch 'upstream/master' b3b0ff1 [Chris Fregly] [SPARK-1981] Add AWS Kinesis streaming support
90 lines
2.7 KiB
Batchfile
90 lines
2.7 KiB
Batchfile
@echo off
|
|
|
|
rem
|
|
rem Licensed to the Apache Software Foundation (ASF) under one or more
|
|
rem contributor license agreements. See the NOTICE file distributed with
|
|
rem this work for additional information regarding copyright ownership.
|
|
rem The ASF licenses this file to You under the Apache License, Version 2.0
|
|
rem (the "License"); you may not use this file except in compliance with
|
|
rem the License. You may obtain a copy of the License at
|
|
rem
|
|
rem http://www.apache.org/licenses/LICENSE-2.0
|
|
rem
|
|
rem Unless required by applicable law or agreed to in writing, software
|
|
rem distributed under the License is distributed on an "AS IS" BASIS,
|
|
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
rem See the License for the specific language governing permissions and
|
|
rem limitations under the License.
|
|
rem
|
|
|
|
set SCALA_VERSION=2.10
|
|
|
|
rem Figure out where the Spark framework is installed
|
|
set FWDIR=%~dp0..\
|
|
|
|
rem Export this as SPARK_HOME
|
|
set SPARK_HOME=%FWDIR%
|
|
|
|
rem Load environment variables from conf\spark-env.cmd, if it exists
|
|
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
|
|
|
|
rem Test that an argument was given
|
|
if not "x%1"=="x" goto arg_given
|
|
echo Usage: run-example ^<example-class^> [example-args]
|
|
echo - set MASTER=XX to use a specific master
|
|
echo - can use abbreviated example class name relative to com.apache.spark.examples
|
|
echo (e.g. SparkPi, mllib.LinearRegression, streaming.KinesisWordCountASL)
|
|
goto exit
|
|
:arg_given
|
|
|
|
set EXAMPLES_DIR=%FWDIR%examples
|
|
|
|
rem Figure out the JAR file that our examples were packaged into.
|
|
set SPARK_EXAMPLES_JAR=
|
|
if exist "%FWDIR%RELEASE" (
|
|
for %%d in ("%FWDIR%lib\spark-examples*.jar") do (
|
|
set SPARK_EXAMPLES_JAR=%%d
|
|
)
|
|
) else (
|
|
for %%d in ("%EXAMPLES_DIR%\target\scala-%SCALA_VERSION%\spark-examples*.jar") do (
|
|
set SPARK_EXAMPLES_JAR=%%d
|
|
)
|
|
)
|
|
if "x%SPARK_EXAMPLES_JAR%"=="x" (
|
|
echo Failed to find Spark examples assembly JAR.
|
|
echo You need to build Spark with sbt\sbt assembly before running this program.
|
|
goto exit
|
|
)
|
|
|
|
rem Set master from MASTER environment variable if given
|
|
if "x%MASTER%"=="x" (
|
|
set EXAMPLE_MASTER=local[*]
|
|
) else (
|
|
set EXAMPLE_MASTER=%MASTER%
|
|
)
|
|
|
|
rem If the EXAMPLE_CLASS does not start with org.apache.spark.examples, add that
|
|
set EXAMPLE_CLASS=%1
|
|
set PREFIX=%EXAMPLE_CLASS:~0,25%
|
|
if not %PREFIX%==org.apache.spark.examples (
|
|
set EXAMPLE_CLASS=org.apache.spark.examples.%EXAMPLE_CLASS%
|
|
)
|
|
|
|
rem Get the tail of the argument list, to skip the first one. This is surprisingly
|
|
rem complicated on Windows.
|
|
set "ARGS="
|
|
:top
|
|
shift
|
|
if "%~1" neq "" (
|
|
set ARGS=%ARGS% "%~1"
|
|
goto :top
|
|
)
|
|
if defined ARGS set ARGS=%ARGS:~1%
|
|
|
|
call "%FWDIR%bin\spark-submit.cmd" ^
|
|
--master %EXAMPLE_MASTER% ^
|
|
--class %EXAMPLE_CLASS% ^
|
|
"%SPARK_EXAMPLES_JAR%" %ARGS%
|
|
|
|
:exit
|