import mill._ import mill.scalalib._ import mill.scalalib.publish._ import $ivy.`com.lihaoyi::mill-contrib-twirllib:`, mill.twirllib._ object astral extends Module { def scalaVersion = "3.3.0" def compile = compiler.compile object compiler extends ScalaModule with PublishModule with TwirlModule { val VERSION = "0.0.1-SNAPSHOT" def scalaVersion = astral.scalaVersion def twirlScalaVersion = scalaVersion def twirlVersion = "1.6.0-RC4" def mainClass = Some("com.astraldb.Astral") def generatedSources = T{ Seq(compileTwirl().classes) } /************************************************* *** Twirl Config *************************************************/ def twirlFormats = super.twirlFormats() ++ Map( "scala" -> "play.twirl.api.TxtFormat" ) /************************************************* *** Backend Dependencies *************************************************/ def ivyDeps = Agg( ivy"com.typesafe.play::twirl-api::${twirlVersion()}" ) def publishVersion = VERSION override def pomSettings = PomSettings( description = "The Astral Compiler Compiler", organization = "com.astraldb", url = "http://astraldb.com", licenses = Seq(License.`Apache-2.0`), versionControl = VersionControl.github("UBOdin", "astral"), developers = Seq( Developer("okennedy", "Oliver Kennedy", "https://odin.cse.buffalo.edu"), ) ) } object catalyst extends ScalaModule { def scalaVersion = astral.scalaVersion def mainClass = Some("com.astraldb.catalyst.Astral") def moduleDeps = Seq(astral.compiler) def outputClasspath = T{ Seq[PathRef](compile().classes) ++ compileClasspath() ++ resources() } def rendered = T { val target = T.dest val files = scala.collection.mutable.Buffer[String]() os.proc( "scala", "-cp", outputClasspath().mkString(":"), "com.astraldb.catalyst.Generate", target ).call( cwd = target, stdout = os.ProcessOutput.Readlines( line => files += line ), stderr = os.ProcessOutput.Readlines( println(_) ) ) for(f <- files) { println(s"GOT : $f") } /* return */ files.map { file => PathRef(target / file) }.toSeq } def render(args: String*) = T.command { for(file <- rendered()) { println(s"\n\n//////////// ${file.path.baseName} ///////////////") println(os.read(file.path)) } } object impl extends ScalaModule { def scalaVersion = "2.13.8" def generatedSources = T{ astral.catalyst.rendered() } def moduleDeps = Seq(astral.compiler, astral.catalyst) def ivyDeps = Agg( ivy"org.apache.spark::spark-sql::3.4.1", ) def internalJavaVersion = T { try { val jvm = System.getProperties().getProperty("java.version") println(f"Running Vizier with `${jvm}`") jvm.split("\\.")(0).toInt } catch { case _:NumberFormatException | _:ArrayIndexOutOfBoundsException => println("Unable to retrieve java version. Guessing 11+") 11 } } def forkArgs = T { if(internalJavaVersion() >= 11){ Seq( // Required on Java 11+ for Arrow compatibility // per: https://spark.apache.org/docs/latest/index.html "-Dio.netty.tryReflectionSetAccessible=true", // Required for Spark on java 11+ // per: https://stackoverflow.com/questions/72230174/java-17-solution-for-spark-java-lang-noclassdeffounderror-could-not-initializ "--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", ) } else { Seq[String]() } } } } }