astral-compiler/build.sc

import mill._
import mill.scalalib._
import mill.scalalib.publish._
import $ivy.`com.lihaoyi::mill-contrib-twirllib:`,  mill.twirllib._
import coursier.maven.{ MavenRepository }

object astral extends Module 
{
  def scalaVersion = "3.3.0"

  def compile = compiler.compile

  object compiler extends ScalaModule 
                  with PublishModule 
                  with TwirlModule
  {
    val VERSION       = "0.0.1-SNAPSHOT"

    def scalaVersion = astral.scalaVersion
    def twirlScalaVersion = scalaVersion
    def twirlVersion = "1.6.0-RC4"

    def mainClass = Some("com.astraldb.Astral")
    def generatedSources = T{ Seq(compileTwirl().classes) }

    /*************************************************
     *** Twirl Config
     *************************************************/
    def twirlFormats = super.twirlFormats() ++ Map(
      "scala" -> "play.twirl.api.TxtFormat"
    )

    /*************************************************
     *** Backend Dependencies
     *************************************************/
    def ivyDeps = Agg(
      ivy"com.typesafe.play::twirl-api::${twirlVersion()}"
    )

    def publishVersion = VERSION
    override def pomSettings = PomSettings(
      description = "The Astral Compiler Compiler",
      organization = "com.astraldb",
      url = "http://astraldb.com",
      licenses = Seq(License.`Apache-2.0`),
      versionControl = VersionControl.github("UBOdin", "astral"),
      developers = Seq(
        Developer("okennedy", "Oliver Kennedy", "https://odin.cse.buffalo.edu"),
      )
    )
  }

  object catalyst extends ScalaModule 
  {
    def scalaVersion = astral.scalaVersion
  
    def mainClass = Some("com.astraldb.catalyst.Astral")

    def moduleDeps = Seq(astral.compiler)

    def outputClasspath = T{ 
      Seq[PathRef](compile().classes) ++ 
      compileClasspath() ++ 
      resources() 
    }

    def rendered = T {
        val target = T.dest
        val files = scala.collection.mutable.Buffer[String]()
        os.proc(
          "scala", 
          "-cp", 
          outputClasspath().mkString(":"), 
          "com.astraldb.catalyst.Generate",
          target
        ).call(
          cwd = target,
          stdout = os.ProcessOutput.Readlines(
            line => files += line
          ),
          stderr = os.ProcessOutput.Readlines(
            println(_)
          )
        )

        for(f <- files)
        {
          println(s"GOT : $f")
        }

        /* return */ 
        files.map { 
          file => PathRef(target / file)
        }.toSeq
      }

    def render(args: String*) = T.command {
      for(file <- rendered())
      {
        println(s"\n\n//////////// ${file.path.baseName} ///////////////")
        println(os.read(file.path))
      }
    }

    object impl extends ScalaModule
    {
      def scalaVersion = "2.12.15"

      def mainClass = Some("com.astraldb.catalyst.TPCHTest")
      def generatedSources = T{ astral.catalyst.rendered() }

      def ivyDeps = Agg(
        ivy"org.apache.spark::spark-sql::3.4.1",
        // ivy"org.apache.commons:commons-compress:1.20",
        ivy"com.lihaoyi::upickle:3.1.2",
        ivy"com.lihaoyi::ujson:0.9.6",
        ivy"info.vizierdb::mimir-caveats::0.3.6"
                          .exclude(
                            "org.slf4j" -> "*",
                            "com.typesafe.play" -> "*",
                            "log4j" -> "*",
                            "org.apache.spark" -> "*"
                          ),
      )

      def repositoriesTask = T.task { super.repositoriesTask() ++ Seq(
        MavenRepository("https://maven.mimirdb.org/"),
      ) }

      def internalJavaVersion = T {
        try {
          val jvm = System.getProperties().getProperty("java.version")
          println(f"Running Catalyst lite with `${jvm}`")
          jvm.split("\\.")(0).toInt
        } catch {
          case _:NumberFormatException | _:ArrayIndexOutOfBoundsException => 
            println("Unable to retrieve java version.  Guessing 11+")
            11
        }
      }

      def forkArgs = T {
        if(internalJavaVersion() >= 11){
          Seq(
            // Required on Java 11+ for Arrow compatibility
            // per: https://spark.apache.org/docs/latest/index.html
            "-Dio.netty.tryReflectionSetAccessible=true",

            // Required for Spark on java 11+
            // per: https://stackoverflow.com/questions/72230174/java-17-solution-for-spark-java-lang-noclassdeffounderror-could-not-initializ
            "--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED",
            "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED",
          )
        } else { Seq[String]() }
      }
    }
  }
}