Simple YAML-backed modules and single-file script modules

ci/test-dist-run.sh

@@ -2,7 +2,7 @@
 
 set -eux
 
-EXAMPLE=example/scalalib/basic/6-realistic
+EXAMPLE=example/scalalib/basic/7-realistic
 
 rm -rf $EXAMPLE/out
 

contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala

@@ -2,7 +2,7 @@ package mill.contrib.scoverage
 
 import coursier.Repository
 import mill._
-import mill.api.{PathRef}
+import mill.api.PathRef
 import mill.api.BuildCtx
 import mill.api.{Result}
 import mill.contrib.scoverage.api.ScoverageReportWorkerApi2.ReportType

core/api/src/mill/api/ExternalModule.scala

@@ -22,7 +22,8 @@ abstract class ExternalModule(implicit
       millFile0
     ) {
 
-  assert(
+  private[mill] def allowNestedExternalModule = false
+  if (!allowNestedExternalModule) assert(
     !" #".exists(millModuleEnclosing0.value.contains(_)),
     "External modules must be at a top-level static path, not " + millModuleEnclosing0.value
   )

core/api/src/mill/api/ModuleCtx.scala

@@ -41,6 +41,7 @@ object ModuleCtx extends LowPriCtx {
   trait Wrapper {
     def moduleCtx: ModuleCtx
     private[mill] def moduleLinearized: Seq[Class[?]]
+    private[mill] def buildOverrides: Map[String, ujson.Value] = Map()
   }
   private case class Impl(
       enclosing: String,

core/api/src/mill/api/internal/RootModule.scala

@@ -26,24 +26,37 @@ abstract class RootModule()(implicit
   // Provided for IDEs to think that one is available and not show errors in
   // build.mill/package.mill even though they can't see the codegen
   def millDiscover: Discover = sys.error("RootModule#millDiscover must be overridden")
+
+  override def buildOverrides: Map[String, ujson.Value] = baseModuleInfo.buildOverrides
 }
 
 @internal
 object RootModule {
   class Info(
       val projectRoot: os.Path,
       val output: os.Path,
-      val topLevelProjectRoot: os.Path
+      val topLevelProjectRoot: os.Path,
+      @com.lihaoyi.unroll val buildOverrides: Map[String, ujson.Value] = Map()
   ) {
+
     def this(
         projectRoot0: String,
         output0: String,
-        topLevelProjectRoot0: String
+        topLevelProjectRoot0: String,
+        headerData: String
     ) = this(
       os.Path(projectRoot0),
       os.Path(output0),
-      os.Path(topLevelProjectRoot0)
+      os.Path(topLevelProjectRoot0),
+      upickle.read[Map[String, ujson.Value]](headerData)
     )
+
+    def this(
+        projectRoot0: String,
+        output0: String,
+        topLevelProjectRoot0: String
+    ) = this(projectRoot0, output0, topLevelProjectRoot0, "{}")
+
     implicit val millMiscInfo: Info = this
   }
 

core/constants/src/mill/constants/Util.java

@@ -5,6 +5,8 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -84,15 +86,20 @@ private static String throwBuildHeaderError(
         + ": " + line + "\n" + msg);
   }
 
-  public static String readBuildHeader(java.nio.file.Path buildFile, String errorFileName) {
+  public static String readBuildHeader(Path buildFile, String errorFileName) {
+    return readBuildHeader(buildFile, errorFileName, false);
+  }
+
+  public static String readBuildHeader(
+      Path buildFile, String errorFileName, boolean allowNonBuild) {
     try {
-      java.util.List<String> lines = java.nio.file.Files.readAllLines(buildFile);
+      java.util.List<String> lines = Files.readAllLines(buildFile);
       boolean readingBuildHeader = true;
       java.util.List<String> output = new ArrayList<>();
       for (int i = 0; i < lines.size(); i++) {
         String line = lines.get(i);
         if (!line.startsWith("//|")) readingBuildHeader = false;
-        else if (!buildFile.getFileName().toString().startsWith("build.")) {
+        else if (!allowNonBuild && !buildFile.getFileName().toString().startsWith("build.")) {
           throwBuildHeaderError(
               errorFileName,
               i,

core/eval/src/mill/eval/EvaluatorImpl.scala

@@ -24,7 +24,11 @@ import mill.resolve.Resolve
 final class EvaluatorImpl private[mill] (
     private[mill] val allowPositionalCommandArgs: Boolean,
     private[mill] val selectiveExecution: Boolean = false,
-    private val execution: Execution
+    private val execution: Execution,
+    scriptModuleResolver: (
+        String,
+        String => Option[mill.Module]
+    ) => Option[Result[mill.api.ExternalModule]]
 ) extends Evaluator {
 
   private[mill] def workspace = execution.workspace
@@ -40,9 +44,18 @@ final class EvaluatorImpl private[mill] (
   def withBaseLogger(newBaseLogger: Logger): Evaluator = new EvaluatorImpl(
     allowPositionalCommandArgs,
     selectiveExecution,
-    execution.withBaseLogger(newBaseLogger)
+    execution.withBaseLogger(newBaseLogger),
+    scriptModuleResolver
   )
 
+  private[mill] def resolveSingleModule(s: String): Option[mill.Module] = {
+    resolveModulesOrTasks(Seq(s), SelectMode.Multi)
+      .toOption
+      .toSeq
+      .flatten
+      .collectFirst { case Left(m) => m }
+  }
+
   /**
    * Takes query selector tokens and resolves them to a list of [[Segments]]
    * representing concrete tasks or modules that match that selector
@@ -59,7 +72,8 @@ final class EvaluatorImpl private[mill] (
         scriptArgs,
         selectMode,
         allowPositionalCommandArgs,
-        resolveToModuleTasks
+        resolveToModuleTasks,
+        scriptModuleResolver = scriptModuleResolver(_, resolveSingleModule)
       )
     }
   }
@@ -75,7 +89,8 @@ final class EvaluatorImpl private[mill] (
         scriptArgs,
         selectMode,
         allowPositionalCommandArgs,
-        resolveToModuleTasks
+        resolveToModuleTasks,
+        scriptModuleResolver = scriptModuleResolver(_, resolveSingleModule)
       )
     }
   }
@@ -97,7 +112,8 @@ final class EvaluatorImpl private[mill] (
           scriptArgs,
           selectMode,
           allowPositionalCommandArgs,
-          resolveToModuleTasks
+          resolveToModuleTasks,
+          scriptModuleResolver = scriptModuleResolver(_, resolveSingleModule)
         )
       }
     }
@@ -115,7 +131,8 @@ final class EvaluatorImpl private[mill] (
           scriptArgs,
           selectMode,
           allowPositionalCommandArgs,
-          resolveToModuleTasks
+          resolveToModuleTasks,
+          scriptModuleResolver = scriptModuleResolver(_, resolveSingleModule)
         )
       }
     }
@@ -276,7 +293,8 @@ final class EvaluatorImpl private[mill] (
             rootModule,
             scriptArgs,
             selectMode,
-            allowPositionalCommandArgs
+            allowPositionalCommandArgs,
+            scriptModuleResolver = scriptModuleResolver(_, resolveSingleModule)
           )
         }
       }

core/exec/src/mill/exec/Execution.scala

@@ -31,7 +31,6 @@ private[mill] case class Execution(
     exclusiveSystemStreams: SystemStreams,
     getEvaluator: () => EvaluatorApi,
     offline: Boolean,
-    headerData: String,
     enableTicker: Boolean
 ) extends GroupExecution with AutoCloseable {
 
@@ -53,7 +52,6 @@ private[mill] case class Execution(
       exclusiveSystemStreams: SystemStreams,
       getEvaluator: () => EvaluatorApi,
       offline: Boolean,
-      headerData: String,
       enableTicker: Boolean
   ) = this(
     baseLogger,
@@ -73,7 +71,6 @@ private[mill] case class Execution(
     exclusiveSystemStreams,
     getEvaluator,
     offline,
-    headerData,
     enableTicker
   )
 

core/exec/src/mill/exec/GroupExecution.scala

@@ -37,42 +37,8 @@ trait GroupExecution {
   def systemExit: ( /* reason */ String, /* exitCode */ Int) => Nothing
   def exclusiveSystemStreams: SystemStreams
   def getEvaluator: () => EvaluatorApi
-  def headerData: String
   def offline: Boolean
 
-  lazy val parsedHeaderData: Map[String, ujson.Value] = {
-    import org.snakeyaml.engine.v2.api.{Load, LoadSettings}
-    val loaded = new Load(LoadSettings.builder().build()).loadFromString(headerData)
-    // recursively convert java data structure to ujson.Value
-    val envWithPwd = env ++ Seq(
-      "PWD" -> workspace.toString,
-      "PWD_URI" -> workspace.toURI.toString,
-      "MILL_VERSION" -> mill.constants.BuildInfo.millVersion,
-      "MILL_BIN_PLATFORM" -> mill.constants.BuildInfo.millBinPlatform
-    )
-    def rec(x: Any): ujson.Value = {
-      import scala.jdk.CollectionConverters._
-      x match {
-        case d: java.util.Date => ujson.Str(d.toString)
-        case s: String => ujson.Str(mill.constants.Util.interpolateEnvVars(s, envWithPwd.asJava))
-        case d: Double => ujson.Num(d)
-        case d: Int => ujson.Num(d)
-        case d: Long => ujson.Num(d)
-        case true => ujson.True
-        case false => ujson.False
-        case null => ujson.Null
-        case m: java.util.Map[Object, Object] =>
-          val scalaMap = m.asScala
-          ujson.Obj.from(scalaMap.map { case (k, v) => (k.toString, rec(v)) })
-        case l: java.util.List[Object] =>
-          val scalaList: collection.Seq[Object] = l.asScala
-          ujson.Arr.from(scalaList.map(rec))
-      }
-    }
-
-    rec(loaded).objOpt.getOrElse(Map.empty[String, ujson.Value]).toMap
-  }
-
   lazy val constructorHashSignatures: Map[String, Seq[(String, Int)]] =
     CodeSigUtils.constructorHashSignatures(codeSignatures)
 
@@ -125,11 +91,28 @@ trait GroupExecution {
     terminal match {
 
       case labelled: Task.Named[_] =>
-        labelled.ctx.segments.value match {
-          case Seq(Segment.Label(single)) if parsedHeaderData.contains(single) =>
-            val jsonData = parsedHeaderData(single)
+
+        // recursively convert java data structure to ujson.Value
+        val envWithPwd = env ++ Seq(
+          "PWD" -> workspace.toString,
+          "PWD_URI" -> workspace.toURI.toString,
+          "MILL_VERSION" -> mill.constants.BuildInfo.millVersion,
+          "MILL_BIN_PLATFORM" -> mill.constants.BuildInfo.millBinPlatform
+        )
+
+        labelled.ctx.segments.last.value match {
+          case single if labelled.ctx.enclosingModule.buildOverrides.contains(single) =>
+            val jsonData = labelled.ctx.enclosingModule.buildOverrides(single)
+
+            import collection.JavaConverters._
+            def rec(x: ujson.Value): ujson.Value = x match {
+              case ujson.Str(s) => mill.constants.Util.interpolateEnvVars(s, envWithPwd.asJava)
+              case ujson.Arr(xs) => ujson.Arr(xs.map(rec))
+              case ujson.Obj(kvs) => ujson.Obj.from(kvs.map((k, v) => (k, rec(v))))
+              case v => v
+            }
             val (resultData, serializedPaths) = PathRef.withSerializedPaths {
-              upickle.read[Any](jsonData)(
+              upickle.read[Any](rec(jsonData))(
                 using labelled.readWriterOpt.get.asInstanceOf[upickle.Reader[Any]]
               )
             }

core/internal/cli/package.mill

@@ -9,8 +9,5 @@ import millbuild.*
 object `package` extends MillPublishScalaModule {
   def moduleDeps = Seq(build.libs.util)
 
-  def mvnDeps = Seq(
-    Deps.millModuledefs,
-    Deps.mainargs
-  )
+  def mvnDeps = Seq(Deps.mainargs)
 }

core/resolve/src/mill/resolve/ParseArgs.scala

@@ -16,13 +16,8 @@ private[mill] object ParseArgs {
 
   /** Separator used in [[SelectMode.Separated]] mode to separate a task-args-tuple from the next target. */
   val TaskSeparator = "+"
-
-  def apply(
-      scriptArgs: Seq[String],
-      selectMode: SelectMode
-  ): Result[Seq[TasksWithParams]] = {
-
-    val MaskPattern = ("""\\+\Q""" + TaskSeparator + """\E""").r
+  val MaskPattern = ("""\\+\Q""" + TaskSeparator + """\E""").r
+  def separate(scriptArgs: Seq[String]) = {
 
     /**
      * Partition the arguments in groups using a separator.
@@ -41,16 +36,13 @@ private[mill] object ParseArgs {
           r2.drop(1)
         )
     }
-    val parts: Seq[Seq[String]] = separated(Seq() /* start value */, scriptArgs)
-    val parsed: Seq[Result[TasksWithParams]] =
-      parts.map(extractAndValidate(_, selectMode == SelectMode.Multi))
-
-    val res1: Result[Seq[TasksWithParams]] = Result.sequence(parsed)
-
-    res1
+    separated(Seq() /* start value */, scriptArgs)
+  }
+  def apply(scriptArgs: Seq[String], selectMode: SelectMode): Seq[Result[TasksWithParams]] = {
+    separate(scriptArgs).map(extractAndValidate(_, selectMode == SelectMode.Multi))
   }
 
-  private def extractAndValidate(
+  def extractAndValidate(
       scriptArgs: Seq[String],
       multiSelect: Boolean
   ): Result[TasksWithParams] = {
@@ -119,7 +111,7 @@ private[mill] object ParseArgs {
       case (h, rest) => Segments(h +: rest)
     }
 
-    P(simpleQuery ~ ("/" ~ simpleQuery.?).? ~ End).map {
+    P(simpleQuery ~ (("/" | ":") ~ simpleQuery.?).? ~ End).map {
       case (q, None) => (None, Some(q))
       case (q, Some(q2)) => (Some(q), q2)
     }