Compare commits
2 Commits
aad5c638cb
...
68aa5fe448
Author | SHA1 | Date |
---|---|---|
Oliver Kennedy | 68aa5fe448 | |
Oliver Kennedy | c17f07b9a9 |
|
@ -1,23 +0,0 @@
|
|||
{
|
||||
"javaSemanticDBVersion": "0.9.8",
|
||||
"semanticDBVersion": "4.8.14",
|
||||
"supportedScalaVersions": [
|
||||
"2.13.12",
|
||||
"2.12.18",
|
||||
"2.12.17",
|
||||
"2.12.16",
|
||||
"2.12.15",
|
||||
"2.13.9",
|
||||
"2.13.10",
|
||||
"2.13.11",
|
||||
"2.11.12",
|
||||
"2.12.11",
|
||||
"2.12.12",
|
||||
"2.12.13",
|
||||
"2.12.14",
|
||||
"2.13.5",
|
||||
"2.13.6",
|
||||
"2.13.7",
|
||||
"2.13.8"
|
||||
]
|
||||
}
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
3609
.bloop/test.json
3609
.bloop/test.json
File diff suppressed because it is too large
Load Diff
|
@ -3,3 +3,5 @@
|
|||
/src/out
|
||||
.bloop
|
||||
.vscode
|
||||
mill-worker-*
|
||||
mill-runner-*
|
||||
|
|
|
@ -1,5 +0,0 @@
|
|||
{
|
||||
"files.watcherExclude": {
|
||||
"**/target": true
|
||||
}
|
||||
}
|
51
build.sc
51
build.sc
|
@ -12,26 +12,43 @@ object mimir_pip extends RootModule with ScalaModule {
|
|||
|
||||
def scalaVersion = "2.12.15"
|
||||
|
||||
def mainClass = Some("org.mimirdb.Pip")
|
||||
def mainClass = Some("org.mimirdb.pip.Main")
|
||||
|
||||
def repositoriesTask = T.task { super.repositoriesTask() ++ Seq(
|
||||
MavenRepository("https://repo.osgeo.org/repository/release/"),
|
||||
)}
|
||||
// def sourceRoots = T.sources(millSourcePath / "bin" / "src")
|
||||
|
||||
def moduleDeps = Seq(lib)
|
||||
|
||||
def ivyDeps = Agg(
|
||||
ivy"org.apache.spark::spark-sql:3.3.1",
|
||||
ivy"org.apache.spark::spark-core:3.3.1",
|
||||
ivy"org.apache.sedona:sedona-common:1.5.0",
|
||||
ivy"org.apache.sedona::sedona-spark-shaded-3.0:1.5.0",
|
||||
ivy"org.datasyslab:geotools-wrapper:1.5.0-28.2"
|
||||
)
|
||||
|
||||
object test extends ScalaTests with TestModule {
|
||||
def ivyDeps = Agg(ivy"org.scalactic::scalactic:3.2.17",
|
||||
ivy"org.scalatest::scalatest:3.2.17"
|
||||
)
|
||||
def testFramework = "org.scalatest.tools.Framework"
|
||||
def repositoriesTask = T.task {
|
||||
super.repositoriesTask() ++
|
||||
lib.repositoriesTask()
|
||||
}
|
||||
|
||||
def test = lib.test
|
||||
|
||||
object lib extends ScalaModule {
|
||||
val VERSION = "0.0.1-SNAPSHOT"
|
||||
|
||||
def scalaVersion = "2.12.15"
|
||||
|
||||
def repositoriesTask = T.task { super.repositoriesTask() ++ Seq(
|
||||
MavenRepository("https://repo.osgeo.org/repository/release/"),
|
||||
)}
|
||||
|
||||
def ivyDeps = Agg(
|
||||
ivy"org.apache.spark::spark-sql:3.3.1",
|
||||
ivy"org.apache.spark::spark-core:3.3.1",
|
||||
ivy"org.apache.sedona:sedona-common:1.5.0",
|
||||
ivy"org.apache.sedona::sedona-spark-shaded-3.0:1.5.0",
|
||||
ivy"org.datasyslab:geotools-wrapper:1.5.0-28.2"
|
||||
)
|
||||
|
||||
object test extends ScalaTests with TestModule {
|
||||
def ivyDeps = Agg(ivy"org.scalactic::scalactic:3.2.17",
|
||||
ivy"org.scalatest::scalatest:3.2.17"
|
||||
)
|
||||
def testFramework = "org.scalatest.tools.Framework"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,10 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
|
||||
import scala.util.Random
|
||||
import java.io.Serializable
|
||||
/* Gaussian object NOTE: Scala uses Camel Case; create a new class file for this UDT and its associated methods */
|
||||
|
||||
|
||||
/* Gaussian object NOTE: Scala uses Camel Case; create a new class file for this UDT and its associated methods */
|
||||
final case class Gauss(mean: Float, sd: Float) {
|
||||
def generate_value():Float = {
|
||||
Random.nextGaussian().asInstanceOf[Float] * sd + mean
|
|
@ -1,4 +1,4 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
|
||||
trait GaussDistribution {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
import org.apache.hadoop.shaded.com.nimbusds.jose.util.StandardCharset
|
||||
|
@ -57,4 +57,4 @@ class GaussType extends UserDefinedType[Gauss]{
|
|||
|
||||
}
|
||||
|
||||
case object GaussType extends org.mimirdb.GaussType with Serializable
|
||||
case object GaussType extends org.mimirdb.pip.GaussType with Serializable
|
|
@ -1,4 +1,4 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
|
||||
import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
|
||||
import org.apache.spark.sql.expressions.Aggregator
|
||||
|
@ -21,7 +21,8 @@ case class DataRow(id: Int, speed: Double, interval: Int, prob: Double)
|
|||
OUT schema is simply the list of bin values (i.e. probability sums for the encoded interval) in order from least
|
||||
speed interval (bin) to greatest.
|
||||
*/
|
||||
object histogram extends Aggregator[DataRow, List[(Int, Double)], Array[Double]] {
|
||||
object Histogram extends Aggregator[DataRow, List[(Int, Double)], Array[Double]]
|
||||
{
|
||||
|
||||
/*
|
||||
ASSUME: The set of tuples being aggregated share the same (observation) point and are thus from the same distribution
|
|
@ -1,11 +1,12 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
|
||||
|
||||
import org.apache.spark.sql.types.UDTRegistration
|
||||
import org.apache.spark.sql.functions
|
||||
|
||||
|
||||
object MimirUDTRegistrator {//by convention uppercase first letter of class/object name
|
||||
object Mimir {
|
||||
//by convention uppercase first letter of class/object name
|
||||
//then just model this after the sedona registrator
|
||||
|
||||
def registerAll(): Unit = {
|
|
@ -1,4 +1,4 @@
|
|||
package org.mimirdb
|
||||
package org.mimirdb.pip
|
||||
|
||||
final case class RandomVariable(i: ()=> Float){
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue