2013-07-30 17:03:15 -04:00
|
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
|
|
<!--
|
|
|
|
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
~ contributor license agreements. See the NOTICE file distributed with
|
|
|
|
~ this work for additional information regarding copyright ownership.
|
|
|
|
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
~ (the "License"); you may not use this file except in compliance with
|
|
|
|
~ the License. You may obtain a copy of the License at
|
|
|
|
~
|
|
|
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
~
|
|
|
|
~ Unless required by applicable law or agreed to in writing, software
|
|
|
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
~ See the License for the specific language governing permissions and
|
|
|
|
~ limitations under the License.
|
|
|
|
-->
|
|
|
|
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
2013-08-31 22:27:07 -04:00
|
|
|
<groupId>org.apache.spark</groupId>
|
2013-07-30 17:03:15 -04:00
|
|
|
<artifactId>spark-parent</artifactId>
|
2014-09-06 18:04:50 -04:00
|
|
|
<version>1.2.0-SNAPSHOT</version>
|
2013-07-30 17:03:15 -04:00
|
|
|
<relativePath>../pom.xml</relativePath>
|
|
|
|
</parent>
|
|
|
|
|
2013-08-31 22:27:07 -04:00
|
|
|
<groupId>org.apache.spark</groupId>
|
2013-12-07 02:15:57 -05:00
|
|
|
<artifactId>spark-mllib_2.10</artifactId>
|
2014-07-10 14:03:37 -04:00
|
|
|
<properties>
|
2014-07-28 15:07:30 -04:00
|
|
|
<sbt.project.name>mllib</sbt.project.name>
|
2014-07-10 14:03:37 -04:00
|
|
|
</properties>
|
2013-07-30 17:03:15 -04:00
|
|
|
<packaging>jar</packaging>
|
|
|
|
<name>Spark Project ML Library</name>
|
2014-03-02 04:00:16 -05:00
|
|
|
<url>http://spark.apache.org/</url>
|
2013-07-30 17:03:15 -04:00
|
|
|
|
|
|
|
<dependencies>
|
2013-08-15 15:10:31 -04:00
|
|
|
<dependency>
|
2013-08-31 22:27:07 -04:00
|
|
|
<groupId>org.apache.spark</groupId>
|
2013-12-15 15:39:58 -05:00
|
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
2013-08-15 15:10:31 -04:00
|
|
|
<version>${project.version}</version>
|
|
|
|
</dependency>
|
Streaming mllib [SPARK-2438][MLLIB]
This PR implements a streaming linear regression analysis, in which a linear regression model is trained online as new data arrive. The design is based on discussions with tdas and mengxr, in which we determined how to add this functionality in a general way, with minimal changes to existing libraries.
__Summary of additions:__
_StreamingLinearAlgorithm_
- An abstract class for fitting generalized linear models online to streaming data, including training on (and updating) a model, and making predictions.
_StreamingLinearRegressionWithSGD_
- Class and companion object for running streaming linear regression
_StreamingLinearRegressionTestSuite_
- Unit tests
_StreamingLinearRegression_
- Example use case: fitting a model online to data from one stream, and making predictions on other data
__Notes__
- If this looks good, I can use the StreamingLinearAlgorithm class to easily implement other analyses that follow the same logic (Ridge, Lasso, Logistic, SVM).
Author: Jeremy Freeman <the.freeman.lab@gmail.com>
Author: freeman <the.freeman.lab@gmail.com>
Closes #1361 from freeman-lab/streaming-mllib and squashes the following commits:
775ea29 [Jeremy Freeman] Throw error if user doesn't initialize weights
4086fee [Jeremy Freeman] Fixed current weight formatting
8b95b27 [Jeremy Freeman] Restored broadcasting
29f27ec [Jeremy Freeman] Formatting
8711c41 [Jeremy Freeman] Used return to avoid indentation
777b596 [Jeremy Freeman] Restored treeAggregate
74cf440 [Jeremy Freeman] Removed static methods
d28cf9a [Jeremy Freeman] Added usage notes
c3326e7 [Jeremy Freeman] Improved documentation
9541a41 [Jeremy Freeman] Merge remote-tracking branch 'upstream/master' into streaming-mllib
66eba5e [Jeremy Freeman] Fixed line lengths
2fe0720 [Jeremy Freeman] Minor cleanup
7d51378 [Jeremy Freeman] Moved streaming loader to MLUtils
b9b69f6 [Jeremy Freeman] Added setter methods
c3f8b5a [Jeremy Freeman] Modified logging
00aafdc [Jeremy Freeman] Add modifiers
14b801e [Jeremy Freeman] Name changes
c7d38a3 [Jeremy Freeman] Move check for empty data to GradientDescent
4b0a5d3 [Jeremy Freeman] Cleaned up tests
74188d6 [Jeremy Freeman] Eliminate dependency on commons
50dd237 [Jeremy Freeman] Removed experimental tag
6bfe1e6 [Jeremy Freeman] Fixed imports
a2a63ad [freeman] Makes convergence test more robust
86220bc [freeman] Streaming linear regression unit tests
fb4683a [freeman] Minor changes for scalastyle consistency
fd31e03 [freeman] Changed logging behavior
453974e [freeman] Fixed indentation
c4b1143 [freeman] Streaming linear regression
604f4d7 [freeman] Expanded private class to include mllib
d99aa85 [freeman] Helper methods for streaming MLlib apps
0898add [freeman] Added dependency on streaming
2014-08-01 23:10:26 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
</dependency>
|
2013-07-30 17:03:15 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
|
|
<artifactId>jetty-server</artifactId>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.jblas</groupId>
|
|
|
|
<artifactId>jblas</artifactId>
|
2014-04-06 20:40:37 -04:00
|
|
|
<version>${jblas.version}</version>
|
2013-07-30 17:03:15 -04:00
|
|
|
</dependency>
|
2014-03-23 20:34:02 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scalanlp</groupId>
|
|
|
|
<artifactId>breeze_${scala.binary.version}</artifactId>
|
2014-08-08 18:07:31 -04:00
|
|
|
<version>0.9</version>
|
2014-06-05 16:13:33 -04:00
|
|
|
<exclusions>
|
|
|
|
<!-- This is included as a compile-scoped dependency by jtransforms, which is
|
|
|
|
a dependency of breeze. -->
|
|
|
|
<exclusion>
|
|
|
|
<groupId>junit</groupId>
|
|
|
|
<artifactId>junit</artifactId>
|
|
|
|
</exclusion>
|
2014-08-01 10:47:44 -04:00
|
|
|
<exclusion>
|
|
|
|
<groupId>org.apache.commons</groupId>
|
|
|
|
<artifactId>commons-math3</artifactId>
|
|
|
|
</exclusion>
|
2014-06-05 16:13:33 -04:00
|
|
|
</exclusions>
|
2014-03-23 20:34:02 -04:00
|
|
|
</dependency>
|
2013-07-30 17:03:15 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.scalatest</groupId>
|
2013-12-15 15:39:58 -05:00
|
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
2013-07-30 17:03:15 -04:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
|
|
|
<dependency>
|
|
|
|
<groupId>org.scalacheck</groupId>
|
2013-12-15 15:39:58 -05:00
|
|
|
<artifactId>scalacheck_${scala.binary.version}</artifactId>
|
2013-07-30 17:03:15 -04:00
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2014-07-30 18:04:33 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>junit</groupId>
|
|
|
|
<artifactId>junit</artifactId>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2013-08-06 18:43:46 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>com.novocode</groupId>
|
|
|
|
<artifactId>junit-interface</artifactId>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2014-08-19 16:28:57 -04:00
|
|
|
<dependency>
|
|
|
|
<groupId>org.apache.spark</groupId>
|
|
|
|
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
|
|
|
|
<version>${project.version}</version>
|
|
|
|
<type>test-jar</type>
|
|
|
|
<scope>test</scope>
|
|
|
|
</dependency>
|
2013-07-30 17:03:15 -04:00
|
|
|
</dependencies>
|
2014-07-11 00:57:54 -04:00
|
|
|
<profiles>
|
|
|
|
<profile>
|
|
|
|
<id>netlib-lgpl</id>
|
|
|
|
<dependencies>
|
|
|
|
<dependency>
|
|
|
|
<groupId>com.github.fommil.netlib</groupId>
|
|
|
|
<artifactId>all</artifactId>
|
|
|
|
<version>1.1.2</version>
|
|
|
|
<type>pom</type>
|
|
|
|
</dependency>
|
|
|
|
</dependencies>
|
|
|
|
</profile>
|
|
|
|
</profiles>
|
2013-07-30 17:03:15 -04:00
|
|
|
<build>
|
2013-12-15 15:39:58 -05:00
|
|
|
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
|
|
|
|
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
|
2013-07-30 17:03:15 -04:00
|
|
|
<plugins>
|
|
|
|
<plugin>
|
|
|
|
<groupId>org.scalatest</groupId>
|
|
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
|
|
</plugin>
|
|
|
|
</plugins>
|
2014-06-26 00:55:49 -04:00
|
|
|
<resources>
|
|
|
|
<resource>
|
|
|
|
<directory>../python</directory>
|
|
|
|
<includes>
|
|
|
|
<include>pyspark/mllib/*.py</include>
|
|
|
|
</includes>
|
|
|
|
</resource>
|
|
|
|
</resources>
|
2013-07-30 17:03:15 -04:00
|
|
|
</build>
|
|
|
|
</project>
|