Try and simplify publishing setup (#3523)

This PR attempts to in-source most of the convenience provided by https://github.com/ckipp01/mill-ci-release into Mill proper. The boilerplate necessary to call `publishAll` on mac laptops or linux github actions environments was trivial and straightforward, so we just try to automate it: 1. Prefix all environment variables with `MILL_`: `MILL_SONATYPE_USERNAME`, `MILL_SONATYPE_PASSWORD`, `MILL_PGP_SECRET_BASE64`, `MILL_PGP_PASSPHRASE` to try and add some rudimentary ownership and namespacing 2. Auto-detect and import `MILL_PGP_SECRET_BASE64`, and auto-detect `MILL_PGP_PASSPHRASE` and include it in the default GPG arguments. `MILL_SONATYPE_USERNAME` and `MILL_SONATYPE_PASSWORD` were already auto-detected 3. Flesh out the `defaultGpgArgs` to allow the optional passphrase to be passed in, and include all the flags that Mill uses in its own CI publishing job 4. Set the defaults for `readTimeout`, `awaitTimeout`, `stagingRelease`, `publishArtifacts` to match the common convention 5. Set `release = true` by default Updated the `ci/release-maven.sh` script to publish using a bootstrap version of Mill, so once this is merged I can immediately test it without needing to go through a rebootstrapping process. The current behavior when passing in flags explicitly should be unchanged, this just acts a lot more aggressively at setting defaults
com-lihaoyi · Sep 12, 2024 · 4d18511 · 4d18511
1 parent a882c63
commit 4d18511
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 57 deletions.
diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml
@@ -31,10 +31,10 @@ jobs:
     concurrency: publish-sonatype-${{ github.sha }}
 
     env:
-      SONATYPE_PGP_SECRET: ${{ secrets.SONATYPE_PGP_SECRET }}
-      SONATYPE_USERNAME: ${{ secrets.SONATYPE_DEPLOY_USER }}
-      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_DEPLOY_PASSWORD }}
-      SONATYPE_PGP_PASSWORD: ${{ secrets.SONATYPE_PGP_PASSWORD }}
+      MILL_SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+      MILL_SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+      MILL_PGP_SECRET_BASE64: ${{ secrets.SONATYPE_PGP_SECRET }}
+      MILL_PGP_PASSPHRASE: ${{ secrets.SONATYPE_PGP_PASSWORD }}
       LANG: "en_US.UTF-8"
       LC_MESSAGES: "en_US.UTF-8"
       LC_ALL: "en_US.UTF-8"

diff --git a/ci/release-maven.sh b/ci/release-maven.sh
@@ -2,22 +2,6 @@
 
 set -eu
 
-echo $SONATYPE_PGP_SECRET | base64 --decode > gpg_key
+./mill -i installLocal
 
-gpg --import  --no-tty --batch --yes gpg_key
-
-rm gpg_key
-
-# Build all artifacts
-./mill -i __.publishArtifacts
-
-# Publish all artifacts
-./mill -i \
-    mill.scalalib.PublishModule/publishAll \
-    --sonatypeCreds $SONATYPE_USERNAME:$SONATYPE_PASSWORD \
-    --gpgArgs --passphrase=$SONATYPE_PGP_PASSWORD,--no-tty,--pinentry-mode,loopback,--batch,--yes,-a,-b \
-    --publishArtifacts __.publishArtifacts \
-    --readTimeout  3600000 \
-    --awaitTimeout 3600000 \
-    --release true \
-    --signed  true
+./target/mill-release -i mill.scalalib.PublishModule/publishAll
diff --git a/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala b/contrib/sonatypecentral/src/mill/contrib/sonatypecentral/SonatypeCentralPublishModule.scala
@@ -13,7 +13,7 @@ import mill.contrib.sonatypecentral.SonatypeCentralPublishModule.{
   getPublishingTypeFromReleaseFlag,
   getSonatypeCredentials
 }
-import mill.scalalib.PublishModule.{defaultGpgArgs, getFinalGpgArgs}
+import mill.scalalib.PublishModule.defaultGpgArgs
 import mill.scalalib.publish.Artifact
 import mill.scalalib.publish.SonatypeHelpers.{
   PASSWORD_ENV_VARIABLE_NAME,
@@ -40,10 +40,13 @@ trait SonatypeCentralPublishModule extends PublishModule {
       val fileMapping = publishData.withConcretePath._1
       val artifact = publishData.meta
       val finalCredentials = getSonatypeCredentials(username, password)()
-
+      PublishModule.pgpImportSecretIfProvided(T.env)
       val publisher = new SonatypeCentralPublisher(
         credentials = finalCredentials,
-        gpgArgs = getFinalGpgArgs(sonatypeCentralGpgArgs()),
+        gpgArgs = sonatypeCentralGpgArgs() match {
+          case "" => PublishModule.defaultGpgArgsForPassphrase(T.env.get("PGP_PASSPHRASE"))
+          case gpgArgs => gpgArgs.split(",").toIndexedSeq
+        },
         connectTimeout = sonatypeCentralConnectTimeout(),
         readTimeout = sonatypeCentralReadTimeout(),
         log = T.log,
@@ -86,10 +89,13 @@ object SonatypeCentralPublishModule extends ExternalModule {
 
     val finalBundleName = if (bundleName.isEmpty) None else Some(bundleName)
     val finalCredentials = getSonatypeCredentials(username, password)()
-
+    PublishModule.pgpImportSecretIfProvided(T.env)
     val publisher = new SonatypeCentralPublisher(
       credentials = finalCredentials,
-      gpgArgs = getFinalGpgArgs(gpgArgs),
+      gpgArgs = gpgArgs match {
+        case "" => PublishModule.defaultGpgArgsForPassphrase(T.env.get("PGP_PASSPHRASE"))
+        case gpgArgs => gpgArgs.split(",").toIndexedSeq
+      },
       connectTimeout = connectTimeout,
       readTimeout = readTimeout,
       log = T.log,

diff --git a/main/src/mill/main/TokenReaders.scala b/main/src/mill/main/TokenReaders.scala
@@ -9,7 +9,7 @@ import mill.resolve.SimpleTaskTokenReader
 case class Tasks[T](value: Seq[mill.define.NamedTask[T]])
 
 object Tasks {
-  private[main] class TokenReader[T]() extends mainargs.TokensReader.Simple[Tasks[T]] {
+  private[mill] class TokenReader[T]() extends mainargs.TokensReader.Simple[Tasks[T]] {
     def shortName = "<tasks>"
     def read(s: Seq[String]): Either[String, Tasks[T]] = {
       Resolve.Tasks.resolve(

diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala
@@ -3,6 +3,7 @@ package scalalib
 
 import mill.define.{Command, ExternalModule, Target, Task}
 import mill.api.{JarManifest, PathRef, Result}
+import mill.main.Tasks
 import mill.scalalib.PublishModule.checkSonatypeCreds
 import mill.scalalib.publish.SonatypeHelpers.{
   PASSWORD_ENV_VARIABLE_NAME,
@@ -220,7 +221,7 @@ trait PublishModule extends JavaModule { outer =>
 
   /**
    * Publish all given artifacts to Sonatype.
-   * Uses environment variables SONATYPE_USERNAME and SONATYPE_PASSWORD as
+   * Uses environment variables MILL_SONATYPE_USERNAME and MILL_SONATYPE_PASSWORD as
    * credentials.
    *
    * @param sonatypeCreds Sonatype credentials in format username:password.
@@ -240,19 +241,21 @@ trait PublishModule extends JavaModule { outer =>
       // TODO: In mill 0.11, we may want to change to a String argument
       // which we can split at `,` symbols, as we do in `PublishModule.publishAll`.
       gpgArgs: Seq[String] = Seq.empty,
-      release: Boolean = false,
-      readTimeout: Int = 60000,
-      connectTimeout: Int = 5000,
-      awaitTimeout: Int = 120 * 1000,
+      release: Boolean = true,
+      readTimeout: Int = 30 * 60 * 1000,
+      connectTimeout: Int = 30 * 60 * 1000,
+      awaitTimeout: Int = 30 * 60 * 1000,
       stagingRelease: Boolean = true
   ): define.Command[Unit] = T.command {
     val PublishModule.PublishData(artifactInfo, artifacts) = publishArtifacts()
+    PublishModule.pgpImportSecretIfProvided(T.env)
     new SonatypePublisher(
       sonatypeUri,
       sonatypeSnapshotUri,
       checkSonatypeCreds(sonatypeCreds)(),
       signed,
-      if (gpgArgs.isEmpty) PublishModule.defaultGpgArgs else gpgArgs,
+      if (gpgArgs.isEmpty) PublishModule.defaultGpgArgsForPassphrase(T.env.get("PGP_PASSPHRASE"))
+      else gpgArgs,
       readTimeout,
       connectTimeout,
       T.log,
@@ -278,7 +281,28 @@ trait PublishModule extends JavaModule { outer =>
 }
 
 object PublishModule extends ExternalModule {
-  val defaultGpgArgs: Seq[String] = Seq("--batch", "--yes", "-a", "-b")
+  val defaultGpgArgs: Seq[String] = defaultGpgArgsForPassphrase(None)
+  def pgpImportSecretIfProvided(env: Map[String, String]): Unit = {
+    for (secret <- env.get("MILL_PGP_SECRET_BASE64")) {
+      os.call(
+        ("gpg", "--import", "--no-tty", "--batch", "--yes"),
+        stdin = java.util.Base64.getDecoder.decode(secret)
+      )
+    }
+  }
+
+  def defaultGpgArgsForPassphrase(passphrase: Option[String]): Seq[String] = {
+    passphrase.map("--passphrase=" + _).toSeq ++
+      Seq(
+        "--no-tty",
+        "--pinentry-mode",
+        "loopback",
+        "--batch",
+        "--yes",
+        "-a",
+        "-b"
+      )
+  }
 
   case class PublishData(meta: Artifact, payload: Seq[(PathRef, String)]) {
 
@@ -297,36 +321,56 @@ object PublishModule extends ExternalModule {
    * Uses environment variables SONATYPE_USERNAME and SONATYPE_PASSWORD as
    * credentials.
    *
+   * @param publishArtifacts what artifacts you want to publish. Defaults to `__.publishArtifacts`
+   *                         which selects all `PublishModule`s in your build
    * @param sonatypeCreds Sonatype credentials in format username:password.
    *                      If specified, environment variables will be ignored.
    *                      <i>Note: consider using environment variables over this argument due
    *                      to security reasons.</i>
-   * @param gpgArgs       GPG arguments. Defaults to `--batch --yes -a -b`.
+   * @param signed
+   * @param gpgArgs       GPG arguments. Defaults to `--passphrase=$MILL_PGP_PASSPHRASE,--no-tty,--pienty-mode,loopback,--batch,--yes,-a,-b`.
    *                      Specifying this will override/remove the defaults.
    *                      Add the default args to your args to keep them.
+   * @param release Whether to release the artifacts after staging them
+   * @param sonatypeUri Sonatype URI to use. Defaults to `oss.sonatype.org`, newer projects
+   *                    may need to set it to https://s01.oss.sonatype.org/service/local
+   * @param sonatypeSnapshotUri Sonatype snapshot URI to use. Defaults to `oss.sonatype.org`, newer projects
+   *                            may need to set it to https://s01.oss.sonatype.org/content/repositories/snapshots
+   * @param readTimeout How long to wait before timing out network reads
+   * @param connectTimeout How long to wait before timing out network connections
+   * @param awaitTimeout How long to wait before timing out on failed uploads
+   * @param stagingRelease
+   * @return
    */
   def publishAll(
-      publishArtifacts: mill.main.Tasks[PublishModule.PublishData],
+      publishArtifacts: Tasks[PublishModule.PublishData] =
+        new Tasks.TokenReader[PublishModule.PublishData]()
+          .read(Seq("__.publishArtifacts"))
+          .getOrElse(sys.error("Unable to resolve __.publishArtifacts")),
       sonatypeCreds: String = "",
       signed: Boolean = true,
-      gpgArgs: String = defaultGpgArgs.mkString(","),
-      release: Boolean = false,
+      gpgArgs: String = "",
+      release: Boolean = true,
       sonatypeUri: String = "https://oss.sonatype.org/service/local",
       sonatypeSnapshotUri: String = "https://oss.sonatype.org/content/repositories/snapshots",
-      readTimeout: Int = 10 * 60 * 1000,
-      connectTimeout: Int = 10 * 60 * 1000,
-      awaitTimeout: Int = 10 * 60 * 1000,
+      readTimeout: Int = 30 * 60 * 1000,
+      connectTimeout: Int = 30 * 60 * 1000,
+      awaitTimeout: Int = 30 * 60 * 1000,
       stagingRelease: Boolean = true
   ): Command[Unit] = T.command {
     val x: Seq[(Seq[(os.Path, String)], Artifact)] = T.sequence(publishArtifacts.value)().map {
       case PublishModule.PublishData(a, s) => (s.map { case (p, f) => (p.path, f) }, a)
     }
+
+    pgpImportSecretIfProvided(T.env)
+
     new SonatypePublisher(
       sonatypeUri,
       sonatypeSnapshotUri,
       checkSonatypeCreds(sonatypeCreds)(),
       signed,
-      getFinalGpgArgs(gpgArgs),
+      if (gpgArgs.isEmpty) defaultGpgArgsForPassphrase(T.env.get("MILL_PGP_PASSPHRASE"))
+      else gpgArgs.split(','),
       readTimeout,
       connectTimeout,
       T.log,
@@ -340,19 +384,11 @@ object PublishModule extends ExternalModule {
     )
   }
 
-  private[mill] def getFinalGpgArgs(initialGpgArgs: String): Seq[String] = {
-    val argsAsString = if (initialGpgArgs.isEmpty) {
-      defaultGpgArgs.mkString(",")
-    } else {
-      initialGpgArgs
-    }
-    argsAsString.split(",").toIndexedSeq
-  }
-
   private def getSonatypeCredsFromEnv: Task[(String, String)] = T.task {
     (for {
-      username <- T.env.get(USERNAME_ENV_VARIABLE_NAME)
-      password <- T.env.get(PASSWORD_ENV_VARIABLE_NAME)
+      // Allow legacy environment variables as well
+      username <- T.env.get(USERNAME_ENV_VARIABLE_NAME).orElse(T.env.get("SONATYPE_USERNAME"))
+      password <- T.env.get(PASSWORD_ENV_VARIABLE_NAME).orElse(T.env.get("SONATYPE_PASSWORD"))
     } yield {
       Result.Success((username, password))
     }).getOrElse(

diff --git a/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala b/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala
@@ -8,8 +8,8 @@ import java.security.MessageDigest
 object SonatypeHelpers {
   // http://central.sonatype.org/pages/working-with-pgp-signatures.html#signing-a-file
 
-  val USERNAME_ENV_VARIABLE_NAME = "SONATYPE_USERNAME"
-  val PASSWORD_ENV_VARIABLE_NAME = "SONATYPE_PASSWORD"
+  val USERNAME_ENV_VARIABLE_NAME = "MILL_SONATYPE_USERNAME"
+  val PASSWORD_ENV_VARIABLE_NAME = "MILL_SONATYPE_PASSWORD"
 
   private[mill] def getArtifactMappings(
       isSigned: Boolean,

diff --git a/scalalib/test/src/mill/scalalib/PublishModuleTests.scala b/scalalib/test/src/mill/scalalib/PublishModuleTests.scala
@@ -163,7 +163,9 @@ object PublishModuleTests extends TestSuite {
           eval.apply(HelloWorldWithPublish.core.checkSonatypeCreds(""))
 
         assert(
-          msg.contains("Consider using SONATYPE_USERNAME/SONATYPE_PASSWORD environment variables")
+          msg.contains(
+            "Consider using MILL_SONATYPE_USERNAME/MILL_SONATYPE_PASSWORD environment variables"
+          )
         )
       }
     }