Documentation updates
This commit is contained in:
parent
30362a21e7
commit
e47e11720f
|
@ -1,28 +1,30 @@
|
|||
require 'fileutils'
|
||||
include FileUtils
|
||||
|
||||
projects = ["core", "examples", "repl", "bagel"]
|
||||
if ENV['SKIP_SCALADOC'] != '1'
|
||||
projects = ["core", "examples", "repl", "bagel"]
|
||||
|
||||
puts "Moving to project root and building scaladoc."
|
||||
curr_dir = pwd
|
||||
cd("..")
|
||||
puts "Moving to project root and building scaladoc."
|
||||
curr_dir = pwd
|
||||
cd("..")
|
||||
|
||||
puts "Running sbt/sbt doc from " + pwd + "; this may take a few minutes..."
|
||||
puts `sbt/sbt doc`
|
||||
puts "Running sbt/sbt doc from " + pwd + "; this may take a few minutes..."
|
||||
puts `sbt/sbt doc`
|
||||
|
||||
puts "moving back into docs dir."
|
||||
cd("docs")
|
||||
puts "moving back into docs dir."
|
||||
cd("docs")
|
||||
|
||||
# Copy over the scaladoc from each project into the docs directory.
|
||||
# This directory will be copied over to _site when `jekyll` command is run.
|
||||
projects.each do |project_name|
|
||||
source = "../" + project_name + "/target/scala-2.9.2/api"
|
||||
dest = "api/" + project_name
|
||||
# Copy over the scaladoc from each project into the docs directory.
|
||||
# This directory will be copied over to _site when `jekyll` command is run.
|
||||
projects.each do |project_name|
|
||||
source = "../" + project_name + "/target/scala-2.9.2/api"
|
||||
dest = "api/" + project_name
|
||||
|
||||
puts "echo making directory " + dest
|
||||
mkdir_p dest
|
||||
puts "echo making directory " + dest
|
||||
mkdir_p dest
|
||||
|
||||
# From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't.
|
||||
puts "cp -r " + source + "/. " + dest
|
||||
cp_r(source + "/.", dest)
|
||||
# From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't.
|
||||
puts "cp -r " + source + "/. " + dest
|
||||
cp_r(source + "/.", dest)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -10,8 +10,9 @@
|
|||
}
|
||||
|
||||
.navbar-inner {
|
||||
padding-top: 5px;
|
||||
padding-bottom: 5px;
|
||||
padding-top: 6px;
|
||||
padding-bottom: 6px;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
body #content {
|
||||
|
@ -35,7 +36,15 @@ h3 {
|
|||
}
|
||||
|
||||
code {
|
||||
color: #333;
|
||||
font-family: "Menlo", "Lucida Console", monospace;
|
||||
background: white;
|
||||
border: none;
|
||||
padding: 0;
|
||||
color: #902000;
|
||||
}
|
||||
|
||||
pre {
|
||||
font-family: "Menlo", "Lucida Console", monospace;
|
||||
}
|
||||
|
||||
.container {
|
||||
|
|
|
@ -11,66 +11,66 @@ Also, I was thrown off for a while at first when I was using markdown
|
|||
code block inside my {% highlight scala %} ... {% endhighlight %} tags
|
||||
(I was using 4 spaces for this), when it turns out that pygments will
|
||||
insert the code (or pre?) tags for you.
|
||||
|
||||
*/
|
||||
|
||||
.hll { background-color: #ffffcc }
|
||||
.c { color: #408080; font-style: italic } /* Comment */
|
||||
.err { border: 1px solid #FF0000 } /* Error */
|
||||
.k { color: #008000; font-weight: bold } /* Keyword */
|
||||
.c { color: #60a0b0; font-style: italic } /* Comment */
|
||||
.err { } /* Error */
|
||||
.k { color: #007020; font-weight: bold } /* Keyword */
|
||||
.o { color: #666666 } /* Operator */
|
||||
.cm { color: #408080; font-style: italic } /* Comment.Multiline */
|
||||
.cp { color: #BC7A00 } /* Comment.Preproc */
|
||||
.c1 { color: #408080; font-style: italic } /* Comment.Single */
|
||||
.cs { color: #408080; font-style: italic } /* Comment.Special */
|
||||
.cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */
|
||||
.cp { color: #007020 } /* Comment.Preproc */
|
||||
.c1 { color: #60a0b0; font-style: italic } /* Comment.Single */
|
||||
.cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */
|
||||
.gd { color: #A00000 } /* Generic.Deleted */
|
||||
.ge { font-style: italic } /* Generic.Emph */
|
||||
.gr { color: #FF0000 } /* Generic.Error */
|
||||
.gh { color: #000080; font-weight: bold } /* Generic.Heading */
|
||||
.gi { color: #00A000 } /* Generic.Inserted */
|
||||
.go { color: #808080 } /* Generic.Output */
|
||||
.gp { color: #000080; font-weight: bold } /* Generic.Prompt */
|
||||
.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
|
||||
.gs { font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #800080; font-weight: bold } /* Generic.Subheading */
|
||||
.gt { color: #0040D0 } /* Generic.Traceback */
|
||||
.kc { color: #008000; font-weight: bold } /* Keyword.Constant */
|
||||
.kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
|
||||
.kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
|
||||
.kp { color: #008000 } /* Keyword.Pseudo */
|
||||
.kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
|
||||
.kt { color: #B00040 } /* Keyword.Type */
|
||||
.m { color: #666666 } /* Literal.Number */
|
||||
.s { color: #BA2121 } /* Literal.String */
|
||||
.na { color: #7D9029 } /* Name.Attribute */
|
||||
.nb { color: #008000 } /* Name.Builtin */
|
||||
.nc { color: #0000FF; font-weight: bold } /* Name.Class */
|
||||
.no { color: #880000 } /* Name.Constant */
|
||||
.nd { color: #AA22FF } /* Name.Decorator */
|
||||
.ni { color: #999999; font-weight: bold } /* Name.Entity */
|
||||
.ne { color: #D2413A; font-weight: bold } /* Name.Exception */
|
||||
.nf { color: #0000FF } /* Name.Function */
|
||||
.nl { color: #A0A000 } /* Name.Label */
|
||||
.nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
|
||||
.nt { color: #008000; font-weight: bold } /* Name.Tag */
|
||||
.nv { color: #19177C } /* Name.Variable */
|
||||
.ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
|
||||
.kc { color: #007020; font-weight: bold } /* Keyword.Constant */
|
||||
.kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
|
||||
.kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
|
||||
.kp { color: #007020 } /* Keyword.Pseudo */
|
||||
.kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
|
||||
.kt { color: #902000 } /* Keyword.Type */
|
||||
.m { color: #40a070 } /* Literal.Number */
|
||||
.s { color: #4070a0 } /* Literal.String */
|
||||
.na { color: #4070a0 } /* Name.Attribute */
|
||||
.nb { color: #007020 } /* Name.Builtin */
|
||||
.nc { color: #0e84b5; font-weight: bold } /* Name.Class */
|
||||
.no { color: #60add5 } /* Name.Constant */
|
||||
.nd { color: #555555; font-weight: bold } /* Name.Decorator */
|
||||
.ni { color: #d55537; font-weight: bold } /* Name.Entity */
|
||||
.ne { color: #007020 } /* Name.Exception */
|
||||
.nf { color: #06287e } /* Name.Function */
|
||||
.nl { color: #002070; font-weight: bold } /* Name.Label */
|
||||
.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
|
||||
.nt { color: #062873; font-weight: bold } /* Name.Tag */
|
||||
.nv { color: #bb60d5 } /* Name.Variable */
|
||||
.ow { color: #007020; font-weight: bold } /* Operator.Word */
|
||||
.w { color: #bbbbbb } /* Text.Whitespace */
|
||||
.mf { color: #666666 } /* Literal.Number.Float */
|
||||
.mh { color: #666666 } /* Literal.Number.Hex */
|
||||
.mi { color: #666666 } /* Literal.Number.Integer */
|
||||
.mo { color: #666666 } /* Literal.Number.Oct */
|
||||
.sb { color: #BA2121 } /* Literal.String.Backtick */
|
||||
.sc { color: #BA2121 } /* Literal.String.Char */
|
||||
.sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
|
||||
.s2 { color: #BA2121 } /* Literal.String.Double */
|
||||
.se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
|
||||
.sh { color: #BA2121 } /* Literal.String.Heredoc */
|
||||
.si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
|
||||
.sx { color: #008000 } /* Literal.String.Other */
|
||||
.sr { color: #BB6688 } /* Literal.String.Regex */
|
||||
.s1 { color: #BA2121 } /* Literal.String.Single */
|
||||
.ss { color: #19177C } /* Literal.String.Symbol */
|
||||
.bp { color: #008000 } /* Name.Builtin.Pseudo */
|
||||
.vc { color: #19177C } /* Name.Variable.Class */
|
||||
.vg { color: #19177C } /* Name.Variable.Global */
|
||||
.vi { color: #19177C } /* Name.Variable.Instance */
|
||||
.il { color: #666666 } /* Literal.Number.Integer.Long */
|
||||
.mf { color: #40a070 } /* Literal.Number.Float */
|
||||
.mh { color: #40a070 } /* Literal.Number.Hex */
|
||||
.mi { color: #40a070 } /* Literal.Number.Integer */
|
||||
.mo { color: #40a070 } /* Literal.Number.Oct */
|
||||
.sb { color: #4070a0 } /* Literal.String.Backtick */
|
||||
.sc { color: #4070a0 } /* Literal.String.Char */
|
||||
.sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
|
||||
.s2 { color: #4070a0 } /* Literal.String.Double */
|
||||
.se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
|
||||
.sh { color: #4070a0 } /* Literal.String.Heredoc */
|
||||
.si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
|
||||
.sx { color: #c65d09 } /* Literal.String.Other */
|
||||
.sr { color: #235388 } /* Literal.String.Regex */
|
||||
.s1 { color: #4070a0 } /* Literal.String.Single */
|
||||
.ss { color: #517918 } /* Literal.String.Symbol */
|
||||
.bp { color: #007020 } /* Name.Builtin.Pseudo */
|
||||
.vc { color: #bb60d5 } /* Name.Variable.Class */
|
||||
.vg { color: #bb60d5 } /* Name.Variable.Global */
|
||||
.vi { color: #bb60d5 } /* Name.Variable.Instance */
|
||||
.il { color: #40a070 } /* Literal.Number.Integer.Long */
|
|
@ -7,7 +7,7 @@ title: Spark Overview
|
|||
TODO(andyk): Rewrite to make the Java API a first class part of the story.
|
||||
{% endcomment %}
|
||||
|
||||
Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It is written in [Scala](http://www.scala-lang.org), a high-level language for the JVM, and exposes a clean language-integrated syntax that makes it easy to write parallel jobs. Spark runs on top of the [Apache Mesos](http://incubator.apache.org/mesos/) cluster manager, Hadoop YARN, or without an independent resource manager (i.e., in "standalone mode").
|
||||
Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It exposes clean language-integrated APIs in [Scala](http://www.scala-lang.org) and Java, providing a wide array of parallel operations. Spark can run on top of the [Apache Mesos](http://incubator.apache.org/mesos/) cluster manager, Hadoop YARN, or without an independent resource manager ("standalone mode").
|
||||
|
||||
# Downloading
|
||||
|
||||
|
@ -15,19 +15,14 @@ Get Spark by checking out the master branch of the Git repository, using `git cl
|
|||
|
||||
# Building
|
||||
|
||||
Spark requires [Scala 2.9](http://www.scala-lang.org/).
|
||||
In addition, to run Spark on a cluster, you will need to install [Mesos](http://incubator.apache.org/mesos/), using the steps in
|
||||
[Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html). However, if you just want to run Spark on a single machine (possibly using multiple cores),
|
||||
you do not need Mesos.
|
||||
|
||||
To build and run Spark, you will need to have Scala's `bin` directory in your `PATH`,
|
||||
Spark requires [Scala 2.9.2](http://www.scala-lang.org/). You will need to have Scala's `bin` directory in your `PATH`,
|
||||
or you will need to set the `SCALA_HOME` environment variable to point
|
||||
to where you've installed Scala. Scala must be accessible through one
|
||||
of these methods on Mesos slave nodes as well as on the master.
|
||||
of these methods on slave nodes on your cluster.
|
||||
|
||||
Spark uses [Simple Build Tool](https://github.com/harrah/xsbt/wiki), which is bundled with it. To compile the code, go into the top-level Spark directory and run
|
||||
|
||||
sbt/sbt compile
|
||||
sbt/sbt package
|
||||
|
||||
# Testing the Build
|
||||
|
||||
|
@ -44,7 +39,7 @@ thread, or `local[N]` to run locally with N threads. You should start by using `
|
|||
Finally, Spark can be used interactively from a modified version of the Scala interpreter that you can start through
|
||||
`./spark-shell`. This is a great way to learn Spark.
|
||||
|
||||
# A Note About Hadoop
|
||||
# A Note About Hadoop Versions
|
||||
|
||||
Spark uses the Hadoop core library to talk to HDFS and other Hadoop-supported
|
||||
storage systems. Because the HDFS protocol has changed in different versions of
|
||||
|
@ -54,23 +49,29 @@ of `project/SparkBuild.scala`, then rebuilding Spark (`sbt/sbt clean compile`).
|
|||
|
||||
# Where to Go from Here
|
||||
|
||||
* [Spark Programming Guide]({{HOME_PATH}}scala-programming-guide.html): how to get started using Spark, and details on the API
|
||||
Programming guides:
|
||||
* [Spark Programming Guide]({{HOME_PATH}}scala-programming-guide.html): how to get started using Spark, and details on the Scala API
|
||||
* [Java Programming Guide]({{HOME_PATH}}java-programming-guide.html): using Spark from Java
|
||||
|
||||
Deployment guides:
|
||||
* [Running Spark on Amazon EC2]({{HOME_PATH}}ec2-scripts.html): scripts that let you launch a cluster on EC2 in about 5 minutes
|
||||
* [Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html): instructions on how to deploy to a private cluster
|
||||
* [Running Spark on YARN]({{HOME_PATH}}running-on-yarn.html): instructions on how to run Spark on top of a YARN cluster
|
||||
* [Spark Standalone Mode]({{HOME_PATH}}spark-standalone.html): instructions on running Spark without Mesos
|
||||
* [Configuration]({{HOME_PATH}}configuration.html): How to set up and customize Spark via its configuration system.
|
||||
* [Bagel Programming Guide]({{HOME_PATH}}bagel-programming-guide.html): implementation of Google's Pregel on Spark
|
||||
* [Spark Debugger]({{HOME_PATH}}spark-debugger.html): experimental work on a debugger for Spark jobs
|
||||
* [Standalone Deploy Mode]({{HOME_PATH}}spark-standalone.html): launch a standalone cluster quickly without Mesos
|
||||
* [Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html): deploy a private cluster using
|
||||
[Apache Mesos](http://incubator.apache.org/mesos)
|
||||
* [Running Spark on YARN]({{HOME_PATH}}running-on-yarn.html): deploy Spark on top of Hadoop NextGen (YARN)
|
||||
|
||||
Miscellaneous:
|
||||
* [Configuration]({{HOME_PATH}}configuration.html): customize Spark via its configuration system.
|
||||
* [Bagel]({{HOME_PATH}}bagel-programming-guide.html): an implementation of Google's Pregel on Spark
|
||||
* [Contributing to Spark](contributing-to-spark.html)
|
||||
|
||||
# Other Resources
|
||||
|
||||
* [Spark Homepage](http://www.spark-project.org)
|
||||
* [AMP Camp](http://ampcamp.berkeley.edu/) - In 2012, the AMP Lab hosted the first AMP Camp which featured talks and hands-on exercises about Spark, Shark, Mesos, and more. [Videos, slides](http://ampcamp.berkeley.edu/agenda) and the [exercises](http://ampcamp.berkeley.edu/exercises) are all available online now. Going through the videos and exercises is a great way to sharpen your Spark skills.
|
||||
* [AMP Camp](http://ampcamp.berkeley.edu/) - In 2012, the AMP Lab hosted the first AMP Camp which featured talks and hands-on exercises about Spark, Shark, Mesos, and more. [Videos, slides](http://ampcamp.berkeley.edu/agenda) and the [exercises](http://ampcamp.berkeley.edu/exercises) are all available online, and provide a great introduction to Spark.
|
||||
* [Paper describing the programming model](http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf)
|
||||
* [Code Examples](http://spark-project.org/examples.html) (more also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of the Spark codebase)
|
||||
* [Mailing List](http://groups.google.com/group/spark-users)
|
||||
* [Code Examples](http://spark-project.org/examples.html): more are also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of Spark
|
||||
* [Mailing List](http://groups.google.com/group/spark-users): ask here for help
|
||||
|
||||
# Community
|
||||
|
||||
|
|
Loading…
Reference in a new issue