[Ready for EngSys Review] ESRP Publishing (#3187)

> [!WARNING]  
> SERVICE OWNERS: no need to review this PR. I'm making engsys changes
to ci.yml files.

Really fixes #3035

[keeping as draft to avoid notifying everyone]

A new release crate-packing strategy for Rust which uses `cargo publish
--dry-run` to produce the crates:

```
cargo publish --dry-run --package <package1> --package <package2> ...
```

This uses existing tooling to do the build with no vendoring or separate
packing. Release order sorting can be read from the tool output and
doesn't need to be computed from metadata. `cargo publish` also verifies
that dependencies defined in the package's `Cargo.toml` are published
and fails if they are not published.


This also supports the current Pre-GA approach of not incrementing
core's versions after release as the workspace dependency includes

For example: 

If a package depends on `[email protected]` and there is no
`[email protected]` published on crates.io, the command will fail. It's
possible to "atomically" generate packages for releases with unreleased
dependencies if those dependencies are included in the package command
(e.g. `storage_blobs` depending on `storage_common`)

## Formatting: 

Formatting attempts to match reasonable outputs. Redirecting stderr from
`cargo publish` would normally result in red console text. The `cargo
publish` command is also grouped:
https://dev.azure.com/azure-sdk/public/_build/results?buildId=5461494&view=logs&j=b766ebde-1fdb-5f11-1350-46ddc53b23cf&t=7ef29fa0-f532-5653-3473-16dc04608431&l=104

<img width="678" height="188" alt="image"
src="https://github.com/user-attachments/assets/33156db7-4567-4259-b69b-eb1720089ea5"
/>


## Tests: 

| Scenario | Expected Outcome | Links | 
| -------- | ---------------- | ---- |
| Release azure_canary w/o required azure_canary_core (intra-service
dependency) | Fail |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460934&view=results
|
| Release azure_canary and azure_canary_core (intra-service dependency)
| Success |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460932&view=results
|
| Release azure_canary_core | Success |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460854&view=results
|
| Release azure_canary_core which depends on **unreleased** azure_core |
Fail |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460945&view=results
|
| Release azure_canary_core which depends on released azure_core |
Success |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460949&view=results
|
| Manually queued live tests (no release) for Key Vault | Success |
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5460954&view=results
|
| PR pipeline works | Success |
https://dev.azure.com/azure-sdk/public/_build/results?buildId=5460931&view=results
|

Author: danieljurek

eng/pipelines/templates/jobs/pack.yml

@@ -55,15 +55,52 @@ jobs:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       PackageInfoDirectory: $(Build.ArtifactStagingDirectory)/PackageInfo
 
-  - task: Powershell@2
-    displayName: "Pack Crates"
-    condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
-    inputs:
-      pwsh: true
-      filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
-      arguments: >
-        -OutputPath '$(Build.ArtifactStagingDirectory)'
-        -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
+  - ${{ if eq('auto', parameters.ServiceDirectory) }}:
+    - task: Powershell@2
+      displayName: Pack Crates
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
+        arguments: >
+          -OutputPath '$(Build.ArtifactStagingDirectory)'
+          -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
+
+  - ${{ else }}:
+    - pwsh: |
+        $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
+        $isReleaseBuild = $true
+        $artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' }
+
+        if (!$artifactsToBuild) {
+          Write-Host "No packages to release. Building all packages in the service directory with no dependency validation."
+          $artifactsToBuild = $artifacts
+          $isReleaseBuild = $false
+        }
+
+        $packageNames = $artifactsToBuild.name
+
+        Write-Host "##vso[task.setvariable variable=PackageNames]$($packageNames -join ',')"
+        if ($isReleaseBuild) {
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]-Release"
+        } else {
+          Write-Host "##vso[task.setvariable variable=AdditionalPackageParams]"
+        }
+      displayName: Configure crate packing
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+
+    - task: Powershell@2
+      displayName: Pack Crates
+      condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
+      inputs:
+        pwsh: true
+        filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
+        arguments: >
+          -OutputPath '$(Build.ArtifactStagingDirectory)'
+          -PackageNames $(PackageNames)
+          -OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json'
+          $(AdditionalPackageParams)
+
 
   - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
     parameters:

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -14,16 +14,13 @@ parameters:
 - name: DevFeedName
   type: string
   default: 'public/azure-sdk-for-rust'
-- name: Environment
-  type: string
-  default: 'cratesio'
 
 stages:
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - ${{ if in(variables['Build.Reason'], 'Manual', '') }}:
-    - ${{ each artifact in parameters.Artifacts }}:
-      - stage: Release_${{artifact.safeName}}
-        displayName: "Release: ${{artifact.name}}"
+    - ${{ if gt(length(parameters.Artifacts), 0) }}:
+      - stage: Release_Batch
+        displayName: "Releasing: ${{length(parameters.Artifacts)}} crates"
         dependsOn: ${{parameters.DependsOn}}
         condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr'))
         variables:
@@ -50,16 +47,17 @@ stages:
 
           - template: /eng/common/pipelines/templates/steps/retain-run.yml
 
-          - script: |
-              echo "##vso[build.addbuildtag]${{artifact.name}}"
-            displayName: Add build tag '${{artifact.name}}'
+          - ${{ each artifact in parameters.Artifacts }}:
+            - script: |
+                echo "##vso[build.addbuildtag]${{artifact.name}}"
+              displayName: Add build tag '${{artifact.name}}'
 
-          - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
-            parameters:
-              ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
-              PackageRepository: Crates.io
-              ReleaseSha: $(Build.SourceVersion)
-              WorkingDirectory: $(Pipeline.Workspace)/_work
+            - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
+              parameters:
+                ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
+                PackageRepository: Crates.io
+                ReleaseSha: $(Build.SourceVersion)
+                WorkingDirectory: $(Pipeline.Workspace)/_work
 
         - deployment: PublishPackage
           displayName: "Publish to Crates.io"
@@ -71,7 +69,10 @@ stages:
             - input: pipelineArtifact  # Required, type of the input artifact
               artifactName: ${{parameters.PipelineArtifactName}}  # Required, name of the pipeline artifact
               targetPath: $(Pipeline.Workspace)/drop  # Optional, specifies where the artifact is downloaded to
-          environment: ${{parameters.Environment}}
+          ${{if parameters.TestPipeline}}:
+            environment: none
+          ${{else}}:
+            environment: package-publish
           # This timeout shouldn't be necessary once we're able to parallelize better. Right now,
           # this is here to ensure larger areas (30+) libraries don't time out.
           timeoutInMinutes: 120
@@ -84,33 +85,71 @@ stages:
             runOnce:
               deploy:
                 steps:
-                - template: /eng/pipelines/templates/steps/use-rust.yml@self
-                  parameters:
-                    Toolchain: stable
-
-                - pwsh: |
-                    $additionalOwners = @('heaths', 'hallipr')
-                    $token = $env:CARGO_REGISTRY_TOKEN
-                    $crateName = '${{artifact.name}}'
-
-                    $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml"
-                    Write-Host "> cargo publish --manifest-path `"$manifestPath`""
-                    cargo publish --manifest-path $manifestPath
-                    if (!$?) {
-                      Write-Error "Failed to publish package: '$crateName'"
-                      exit 1
-                    }
-
-                    $existingOwners = (cargo owner --list $crateName) -replace " \(.*", ""
-                    $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ }
-
-                    foreach ($owner in $missingOwners) {
-                      Write-Host "> cargo owner --add $owner $crateName"
-                      cargo owner --add $owner $crateName
-                    }
-                  displayName: Publish Crate
-                  env:
-                    CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
+                # This loop over artifacts is used to produce the correct number
+                # of ESRP release tasks. It has the side effect of also setting
+                # the artifact name by looking up the index of the current
+                # "artifact.name" in the parameters.Artifacts array, using that
+                # as an "index" and then using that same index to look up the
+                # actual artifact to release in the release-order.json file.
+                - ${{ each artifact in parameters.Artifacts }}:
+                  - pwsh: |
+                      # From the DevOps template artifact loop calculate the current index
+                      $indexItem = '${{ artifact.name }}'
+                      [array] $indexList = ConvertFrom-Json '${{ convertToJson(parameters.Artifacts.*.name) }}'
+                      $index = $indexList.IndexOf($indexItem)
+                      Write-Host "Index of template artifact: $index"
+
+                      [array] $artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json
+
+                      $artifactName = $artifacts[$index]
+
+                      Write-Host "Releasing artifact: $artifactName"
+
+                      $artifactRootPath = '$(Pipeline.Workspace)/drop'
+                      $outDir = '$(Pipeline.Workspace)/esrp-release'
+
+                      if (Test-Path $outDir) {
+                        Write-Host "Cleaning output directory: $outDir"
+                        Remove-Item -Path $outDir -Recurse -Force
+                      }
+                      New-Item -ItemType Directory -Path $outDir -Force | Out-Null
+
+                      $packageMetadataPath = "$artifactRootPath/PackageInfo/$artifactName.json"
+                      if (!(Test-Path $packageMetadataPath)) {
+                        Write-Error "Package metadata file not found: $packageMetadataPath"
+                        exit 1
+                      }
+
+                      $packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json
+                      $packageVersion = $packageMetadata.version
+                      Write-Host "Package version: $packageVersion"
+
+                      $cratePath = "$artifactRootPath/$artifactName/$artifactName-$packageVersion.crate"
+                      Copy-Item `
+                        -Path $cratePath `
+                        -Destination $outDir
+                      Write-Host "Contents of $outDir"
+                      Get-ChildItem -Path $outDir | ForEach-Object { Write-Host $_.FullName }
+                    displayName: 'Copy crate for ESRP'
+
+                  - task: EsrpRelease@10
+                    displayName: 'ESRP Release'
+                    inputs:
+                      connectedservicename: 'Azure SDK PME Managed Identity'
+                      ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b'
+                      DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
+                      Usemanagedidentity: true
+                      KeyVaultName: 'kv-azuresdk-codesign'
+                      SignCertName: 'azure-sdk-esrp-release-certificate'
+                      intent: 'packagedistribution'
+                      contenttype: 'Rust'
+                      contentsource: 'Folder'
+                      folderlocation: '$(Pipeline.Workspace)/esrp-release'
+                      waitforreleasecompletion: true
+                      owners: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
+                      approvers: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
+                      serviceendpointurl: 'https://api.esrp.microsoft.com/'
+                      mainpublisher: 'ESRPRELPACMANTEST'
 
         - job: UpdatePackageVersion
           displayName: "API Review and Package Version Update"
@@ -130,69 +169,32 @@ stages:
             displayName: Download ${{parameters.PipelineArtifactName}} artifact
             artifact: ${{parameters.PipelineArtifactName}}
 
-          - template: /eng/common/pipelines/templates/steps/create-apireview.yml
-            parameters:
-              ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
-              Artifacts: ${{parameters.Artifacts}}
-              ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
-              MarkPackageAsShipped: true
-              ArtifactName: ${{parameters.PipelineArtifactName}}
-              SourceRootPath: $(System.DefaultWorkingDirectory)
-              PackageName: ${{artifact.name}}
-
-          # Apply the version increment to each library, which updates the Cargo.toml and changelog files.
-          - task: PowerShell@2
-            displayName: Increment ${{artifact.name}} version
-            inputs:
-              targetType: filePath
-              filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
-              arguments: >
-                -ServiceDirectory '${{parameters.ServiceDirectory}}'
-                -PackageName '${{artifact.name}}'
+          - ${{ each artifact in parameters.Artifacts }}:
+            - template: /eng/common/pipelines/templates/steps/create-apireview.yml
+              parameters:
+                ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
+                Artifacts: ${{parameters.Artifacts}}
+                ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
+                MarkPackageAsShipped: true
+                ArtifactName: ${{parameters.PipelineArtifactName}}
+                SourceRootPath: $(System.DefaultWorkingDirectory)
+                PackageName: ${{artifact.name}}
+
+            # Apply the version increment to each library, which updates the Cargo.toml and changelog files.
+            - task: PowerShell@2
+              displayName: Increment ${{artifact.name}} version
+              inputs:
+                targetType: filePath
+                filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
+                arguments: >
+                  -ServiceDirectory '${{parameters.ServiceDirectory}}'
+                  -PackageName '${{artifact.name}}'
 
           - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
             parameters:
               PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId)
-              CommitMsg: "Increment package version after release of ${{ artifact.name }}"
+              CommitMsg: "Increment package version after release of ${{ join(', ', parameters.Artifacts.*.name) }}"
               PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases"
               CloseAfterOpenForTesting: '${{parameters.TestPipeline}}'
               ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
                 BaseBranchName: main
-
-        - ${{ if eq(parameters.TestPipeline, true) }}:
-          - job: ManualApproval
-            displayName: "Manual approval"
-            dependsOn: PublishPackage
-            condition: ne(variables['Skip.PublishPackage'], 'true')
-            pool: server
-            timeoutInMinutes: 120 # 2 hours
-            steps:
-            - task: ManualValidation@1
-              timeoutInMinutes: 60 # 1 hour
-              inputs:
-                notifyUsers: '' # Required, but empty string allowed
-                allowApproversToApproveTheirOwnRuns: true
-                instructions: "Approve yank of ${{ artifact.name }}"
-                onTimeout: 'resume'
-
-          - job: YankCrates
-            displayName: "Yank Crates"
-            dependsOn: ManualApproval
-            condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true'))
-            steps:
-            - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-
-            - download: current
-              displayName: Download ${{parameters.PipelineArtifactName}} artifact
-              artifact: ${{parameters.PipelineArtifactName}}
-
-            - task: PowerShell@2
-              displayName: Yank Crates
-              env:
-                CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
-              inputs:
-                targetType: filePath
-                filePath: $(Build.SourcesDirectory)/eng/scripts/Yank-Crates.ps1
-                arguments:
-                  -CrateNames '${{artifact.name}}'
-                  -PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo'

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -148,6 +148,9 @@ extends:
             parameters:
               DependsOn: "Build"
               ServiceDirectory: ${{ parameters.ServiceDirectory }}
-              Artifacts: ${{ parameters.Artifacts }}
+              Artifacts: 
+                - ${{ each artifact in parameters.Artifacts }}: 
+                  - ${{ if ne(artifact.releaseInBatch, 'false')}}: 
+                    - ${{ artifact }}
               TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }}
               PipelineArtifactName: packages

eng/scripts/Language-Settings.ps1

@@ -1,7 +1,7 @@
 $Language = "rust"
 $LanguageDisplayName = "Rust"
 $PackageRepository = "crates.io"
-$packagePattern = "Cargo.toml"
+$packagePattern = "*.crate"
 #$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv"
 $GithubUri = "https://github.com/Azure/azure-sdk-for-rust"
 $PackageRepositoryUri = "https://crates.io/crates"
@@ -139,15 +139,32 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
   return $additionalPackages ?? @()
 }
 
+# $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
 function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
-  #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory
-  $package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json
+  # Create a temporary folder for extraction
+  $extractionPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
+  New-Item -ItemType Directory -Path $extractionPath | Out-Null
+
+  # Extract the .crate file (which is a tarball) to the temporary folder
+  tar -xvf $pkg.FullName -C $extractionPath
+  $cargoTomlPath = [System.IO.Path]::Combine($extractionPath, $pkg.BaseName, 'Cargo.toml')
+
+  Write-Host "Reading package info from $cargoTomlPath"
+  if (!(Test-Path $cargoTomlPath)) {
+    $message = "The Cargo.toml file was not found in the package artifact at $cargoTomlPath"
+    LogError $message
+    throw $message
+  }
+
+  $package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json
 
   $packageName = $package.name
   $packageVersion = $package.version
 
-  $changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1
-  $readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1
+  $packageAssetPath = [System.IO.Path]::Combine($extractionPath, "$packageName-$packageVersion")
+
+  $changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1
+  $readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1
 
   if ($changeLogLoc) {
     $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion