diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml
index 81c74e66d29..127b2195016 100644
--- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml
+++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml
@@ -58,13 +58,20 @@ variables:
value: ${{ parameters.ReleaseTagVar }}
- name: SKIP_SIGNING
value: ${{ parameters.SKIP_SIGNING }}
- - group: 'AzDevOpsArtifacts'
- - group: 'mscodehub-feed-read-akv'
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
+ - name: ENABLE_MSBUILD_BINLOGS
+ value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }}
extends:
template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
parameters:
customTags: 'ES365AIMigrationTooling'
+ featureFlags:
+ LinuxHostVersion:
+ Network: KS3
+ WindowsHostVersion:
+ Network: KS3
globalSdl:
disableLegacyManifest: true
# disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates.
@@ -107,15 +114,18 @@ extends:
type: windows
variables:
- - name: ob_sdl_tsa_configFile
- value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- - name: ob_sdl_credscan_suppressionsFile
- value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- - name: ob_sdl_codeql_compiled_enabled
- value: true
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson'
+ - name: ob_sdl_codeSignValidation_enabled
+ value: false
+ - name: ob_sdl_codeql_compiled_enabled
+ value: false
+ - name: ob_sdl_credscan_suppressionsFile
+ value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
+ - name: ob_sdl_tsa_configFile
+ value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
+ - name: ob_signing_setup_enabled
+ value: false
steps:
- checkout: self
@@ -126,6 +136,8 @@ extends:
- pwsh: |
Get-ChildItem Env:
displayName: Capture environment variables
+ env:
+ ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
- template: /.pipelines/templates/SetVersionVariables.yml@self
parameters:
diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml
index c70da261cc5..e2f71096e68 100644
--- a/.pipelines/PowerShell-Packages-Official.yml
+++ b/.pipelines/PowerShell-Packages-Official.yml
@@ -52,6 +52,7 @@ variables:
value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project
- name: LinuxContainerImage
value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+ - group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: branchCounterKey
value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])]
diff --git a/.pipelines/templates/checkAzureContainer.yml b/.pipelines/templates/checkAzureContainer.yml
index dae78a7e66a..a5ce2b1c666 100644
--- a/.pipelines/templates/checkAzureContainer.yml
+++ b/.pipelines/templates/checkAzureContainer.yml
@@ -13,9 +13,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- - name: ob_sdl_codeql_compiled_enabled
- value: true
+ - name: ob_sdl_codeql_compiled_enabled
+ value: false
displayName: Delete blob is exists
pool:
diff --git a/.pipelines/templates/insert-nuget-config-azfeed.yml b/.pipelines/templates/insert-nuget-config-azfeed.yml
index fef9f3f4012..20057440c00 100644
--- a/.pipelines/templates/insert-nuget-config-azfeed.yml
+++ b/.pipelines/templates/insert-nuget-config-azfeed.yml
@@ -1,39 +1,53 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
+- name: "ob_restore_phase"
+ type: boolean
+ default: true
+
steps:
+- task: NuGetAuthenticate@1
+ displayName: Install Azure Artifacts Credential Provider
+ inputs:
+ forceReinstallCredentialProvider: true
+
- pwsh: |
- $configPath = "${env:NugetConfigDir}/nuget.config"
- Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
+ try {
+ $configPath = "${env:NugetConfigDir}/nuget.config"
+ Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
- $powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
+ Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'"
+ Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'
- New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
- if(-not (Test-Path $configPath))
- {
- throw "nuget.config is not created"
+ if(-not (Test-Path $configPath))
+ {
+ throw "nuget.config is not created"
+ }
}
- Get-Content $configPath | Write-Verbose -Verbose
- displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
- condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
+ catch {
+ Get-Error
+ throw
+ }
+ displayName: 'Switch to production Azure DevOps feed for all nuget.configs'
+ condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
- ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
+ ob_restore_phase: ${{ parameters.ob_restore_phase }}
- pwsh: |
- $configPath = "${env:NugetConfigDir}/nuget.config"
- Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
-
- $powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
-
- New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
- if (-not (Test-Path $configPath))
- {
- throw "nuget.config is not created"
+ Get-ChildItem ${{ parameters.repoRoot }}/nuget.config -Recurse | Foreach-Object {
+ Write-Verbose -Verbose "--- START $($_.fullname) ---"
+ get-content $_.fullname | Out-String -width 9999 -Stream | write-Verbose -Verbose
+ Write-Verbose -Verbose "--- END $($_.fullname) ---"
}
- Get-Content $configPath | Write-Verbose -Verbose
- displayName: 'Add nuget.config for Azure DevOps feed for packages'
- condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
+ displayName: 'Capture all nuget.config files'
+ condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
+ env:
+ ob_restore_phase: ${{ parameters.ob_restore_phase }}
+
+- pwsh: |
+ Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose
+ displayName: Capture VSS* Environment
+ condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
- NugetConfigDir: ${{ parameters.repoRoot }}
- ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
+ ob_restore_phase: ${{ parameters.ob_restore_phase }}
diff --git a/.pipelines/templates/mac.yml b/.pipelines/templates/mac.yml
index 424471ded13..69ff4688854 100644
--- a/.pipelines/templates/mac.yml
+++ b/.pipelines/templates/mac.yml
@@ -20,10 +20,9 @@ jobs:
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
- - name: ob_sdl_binskim_enabled
- value: true
- - name: ob_sdl_credscan_suppressionsfileforartifacts
- value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json
+ - name: PowerShellRoot
+ value: $(Build.SourcesDirectory)
+
steps:
- checkout: self
clean: true
@@ -39,19 +38,19 @@ jobs:
# make the current user the owner
sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell"
displayName: 'Create $(Agent.TempDirectory)/PowerShell'
- - template: /.pipelines/templates/cloneToOfficialPath.yml@self
- parameters:
- nativePathRoot: '$(Agent.TempDirectory)'
+
- pwsh: |
- tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
+ Import-Module $(PowerShellRoot)/build.psm1 -Force
+ Start-PSBootstrap -Package
displayName: 'Bootstrap VM'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
+
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- pwsh: |
- $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
+ $env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
@@ -76,6 +75,7 @@ jobs:
displayName: 'Build'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
+
- template: /.pipelines/templates/step/finalize.yml@self
- job: sign_${{ parameters.buildArchitecture }}
diff --git a/.pipelines/templates/nupkg.yml b/.pipelines/templates/nupkg.yml
index e708eac808b..34e313723bb 100644
--- a/.pipelines/templates/nupkg.yml
+++ b/.pipelines/templates/nupkg.yml
@@ -22,7 +22,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- - group: 'AzDevOpsArtifacts'
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
@@ -89,23 +90,9 @@ jobs:
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- - pwsh: |
- $repoRoot = "$(PowerShellRoot)"
- Write-Verbose -Verbose "repoRoot: $repoRoot"
-
- $configPath = "$repoRoot/nuget.config"
- Import-Module "$repoRoot/build.psm1" -Force
- New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)"
-
- if(-not (Test-Path $configPath))
- {
- throw "nuget.config is not created"
- }
- Get-Content $configPath | Write-Verbose -Verbose
- displayName: 'Add nuget.config for Azure DevOps feed for packages'
- condition: and(succeededOrFailed(), ne(variables['PowerShellCore_PublicPackages'], ''))
- env:
- ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
+ - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
+ parameters:
+ repoRoot: $(PowerShellRoot)
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
diff --git a/.pipelines/templates/release-MakeBlobPublic.yml b/.pipelines/templates/release-MakeBlobPublic.yml
index 11b45733270..1e3789b207a 100644
--- a/.pipelines/templates/release-MakeBlobPublic.yml
+++ b/.pipelines/templates/release-MakeBlobPublic.yml
@@ -30,9 +30,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- - name: ob_sdl_codeql_compiled_enabled
- value: true
+ - name: ob_sdl_codeql_compiled_enabled
+ value: false
steps:
- checkout: self
diff --git a/.pipelines/templates/release-validate-sdk.yml b/.pipelines/templates/release-validate-sdk.yml
index 865eca964bb..db3f550b965 100644
--- a/.pipelines/templates/release-validate-sdk.yml
+++ b/.pipelines/templates/release-validate-sdk.yml
@@ -19,7 +19,8 @@ jobs:
- ImageOverride -equals ${{ parameters.imageName }}
variables:
- - group: AzDevOpsArtifacts
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
diff --git a/.pipelines/templates/testartifacts.yml b/.pipelines/templates/testartifacts.yml
index d3e2f254887..039e9336d7c 100644
--- a/.pipelines/templates/testartifacts.yml
+++ b/.pipelines/templates/testartifacts.yml
@@ -22,15 +22,22 @@ jobs:
steps:
- checkout: self
clean: true
+ env:
+ ob_restore_phase: true
+
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
- repoRoot: $(Build.SourcesDirectory)
+ repoRoot: $(Build.SourcesDirectory)/PowerShell
+ ob_restore_phase: true
+
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
+ ob_restore_phase: true
+
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@@ -58,6 +65,13 @@ jobs:
BuildTestPackage -runtime win-arm64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
+ env:
+ ob_restore_phase: true
+
+ - pwsh: |
+ Write-Host "This doesn't do anything but make the build phase run."
+ displayName: Dummy build task
+
- job: build_testartifacts_nonwin
variables:
@@ -75,15 +89,22 @@ jobs:
steps:
- checkout: self
clean: true
+ env:
+ ob_restore_phase: true
+
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
- repoRoot: $(Build.SourcesDirectory)
+ repoRoot: $(Build.SourcesDirectory)/PowerShell
+ ob_restore_phase: true
+
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
+ ob_restore_phase: true
+
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@@ -113,3 +134,9 @@ jobs:
BuildTestPackage -runtime linux-musl-x64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
+ env:
+ ob_restore_phase: true
+
+ - pwsh: |
+ Write-Host "This doesn't do anything but make the build phase run."
+ displayName: Dummy build task
diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml
index 150fef9ef71..83c3a91d267 100644
--- a/.pipelines/templates/uploadToAzure.yml
+++ b/.pipelines/templates/uploadToAzure.yml
@@ -21,9 +21,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- - name: ob_sdl_codeql_compiled_enabled
- value: true
+ - name: ob_sdl_codeql_compiled_enabled
+ value: false
steps:
- checkout: self
diff --git a/.vsts-ci/templates/ci-build.yml b/.vsts-ci/templates/ci-build.yml
index e64ab4967a2..e8748f9756f 100644
--- a/.vsts-ci/templates/ci-build.yml
+++ b/.vsts-ci/templates/ci-build.yml
@@ -20,7 +20,7 @@ jobs:
displayName: Set Build Name for Non-PR
condition: ne(variables['Build.Reason'], 'PullRequest')
- - ${{ if ne(variables['AzDevOpsFeed'], '') }}:
+ - ${{ if ne(variables['UseAzDevOpsFeed'], '') }}:
- template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
- pwsh: |
diff --git a/build.psm1 b/build.psm1
index d5740890211..c069b7a3bf5 100644
--- a/build.psm1
+++ b/build.psm1
@@ -680,6 +680,8 @@ Fix steps:
if ($config.Count -gt 0) {
$configPublishPath = Join-Path -Path $publishPath -ChildPath "powershell.config.json"
Set-Content -Path $configPublishPath -Value ($config | ConvertTo-Json) -Force -ErrorAction Stop
+ } else {
+ Write-Warning "No powershell.config.json generated for $publishPath"
}
# Restore the Pester module
@@ -701,25 +703,47 @@ Fix steps:
function Switch-PSNugetConfig {
param(
+ [Parameter(Mandatory = $true, ParameterSetName = 'user')]
+ [Parameter(Mandatory = $true, ParameterSetName = 'nouser')]
[ValidateSet('Public', 'Private')]
- [string] $Source = 'Public'
+ [string] $Source,
+
+ [Parameter(Mandatory = $true, ParameterSetName = 'user')]
+ [string] $UserName,
+
+ [Parameter(Mandatory = $true, ParameterSetName = 'user')]
+ [string] $ClearTextPAT
)
+ Clear-PipelineNugetAuthentication
+
+ $extraParams = @()
+ if ($UserName) {
+ $extraParams = @{
+ UserName = $UserName
+ ClearTextPAT = $ClearTextPAT
+ }
+ }
+
if ( $Source -eq 'Public') {
$dotnetSdk = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v2'; Name = 'dotnet' }
$gallery = [NugetPackageSource] @{Url = 'https://www.powershellgallery.com/api/v2/'; Name = 'psgallery' }
$nugetorg = [NugetPackageSource] @{Url = 'https://api.nuget.org/v3/index.json'; Name = 'nuget.org' }
- New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/"
- New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/"
+ New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/" @extraParams
+ New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/" @extraParams
} elseif ( $Source -eq 'Private') {
- $powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json'; Name = 'powershell' }
+ $powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/PowerShell/nuget/v3/index.json'; Name = 'powershell' }
- New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/"
- New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/"
+ New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/" @extraParams
+ New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/" @extraParams
} else {
throw "Unknown source: $Source"
}
+
+ if ($UserName -or $ClearTextPAT) {
+ Set-PipelineNugetAuthentication
+ }
}
function Restore-PSPackage
@@ -781,6 +805,10 @@ function Restore-PSPackage
$RestoreArguments += "--interactive"
}
+ if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
+ $RestoreArguments += '-bl'
+ }
+
$ProjectDirs | ForEach-Object {
$project = $_
Write-Log -message "Run dotnet restore $project $RestoreArguments"
@@ -798,6 +826,23 @@ function Restore-PSPackage
$retryCount++
if($retryCount -ge $maxTries)
{
+ if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
+ if ( Test-Path ./msbuild.binlog ) {
+ if (!(Test-Path $env:OB_OUTPUTDIRECTORY -PathType Container)) {
+ $null = New-Item -path $env:OB_OUTPUTDIRECTORY -ItemType Directory -Force -Verbose
+ }
+
+ $projectName = Split-Path -Leaf -Path $project
+ $binlogFileName = "${projectName}.msbuild.binlog"
+ if ($IsMacOS) {
+ $resolvedPath = (Resolve-Path -Path ./msbuild.binlog).ProviderPath
+ Write-Host "##vso[artifact.upload containerfolder=$binLogFileName;artifactname=$binLogFileName]$resolvedPath"
+ } else {
+ Copy-Item -Path ./msbuild.binlog -Destination "$env:OB_OUTPUTDIRECTORY/${projectName}.msbuild.binlog" -Verbose
+ }
+ }
+ }
+
throw
}
continue
@@ -2203,6 +2248,18 @@ function Start-PSBootstrap {
Install-Wix
}
}
+
+ if ($env:TF_BUILD) {
+ Write-Verbose -Verbose "--- Start - Capturing nuget sources"
+ dotnet nuget list source --format detailed
+ Write-Verbose -Verbose "--- End - Capturing nuget sources"
+ }
+
+ if ($env:TF_BUILD) {
+ Write-Verbose -Verbose "--- Start - Capturing nuget sources"
+ dotnet nuget list source --format detailed
+ Write-Verbose -Verbose "--- End - Capturing nuget sources"
+ }
} finally {
Pop-Location
}
@@ -3310,6 +3367,7 @@ function New-NugetPackageSource {
return [NugetPackageSource] @{Url = $Url; Name = $Name }
}
+$script:NuGetEndpointCredentials = [System.Collections.Generic.Dictionary[String,System.Object]]::new()
function New-NugetConfigFile {
param(
[Parameter(Mandatory = $true, ParameterSetName ='user')]
@@ -3353,23 +3411,53 @@ function New-NugetConfigFile {
'@
$content = $nugetConfigHeaderTemplate
+ $feedNamePostfix = ''
+ if ($UserName) {
+ $feedNamePostfix += '-' + $UserName.Replace('@', '-').Replace('.', '-')
+ }
[NugetPackageSource]$source = $null
+ $newLine = [Environment]::NewLine
foreach ($source in $NugetPackageSource) {
- $content += $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name)
+ $content += $newLine + $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name + $feedNamePostfix)
}
- $content += $nugetPackageSourceFooterTemplate
+ $content += $newLine + $nugetPackageSourceFooterTemplate
if ($UserName -or $ClearTextPAT) {
- $content += $nugetCredentialsTemplate.Replace('[USERNAME]', $UserName).Replace('[PASSWORD]', $ClearTextPAT)
+ foreach ($source in $NugetPackageSource) {
+ if (!$script:NuGetEndpointCredentials.ContainsKey($source.Url)) {
+ $script:NuGetEndpointCredentials.Add($source.Url, @{
+ endpoint = $source.Url
+ username = $UserName
+ password = $ClearTextPAT
+ })
+ }
+ }
}
- $content += $nugetConfigFooterTemplate
+ $content += $newLine + $nugetConfigFooterTemplate
Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force
}
+function Clear-PipelineNugetAuthentication {
+ $script:NuGetEndpointCredentials.Clear()
+}
+
+function Set-PipelineNugetAuthentication {
+ $endpointcredentials = @()
+
+ foreach ($key in $script:NuGetEndpointCredentials.Keys) {
+ $endpointcredentials += $script:NuGetEndpointCredentials[$key]
+ }
+
+ $json = @{
+ endpointCredentials = $endpointcredentials
+ } | convertto-json -Compress
+ Set-PipelineVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $json
+}
+
function Set-CorrectLocale
{
if (-not $IsLinux)
@@ -3483,3 +3571,19 @@ function Clear-NativeDependencies
$deps | ConvertTo-Json -Depth 20 | Set-Content "$PublishFolder/pwsh.deps.json" -Force
}
+
+function Set-PipelineVariable {
+ param(
+ [parameter(Mandatory)]
+ [string] $Name,
+ [parameter(Mandatory)]
+ [string] $Value
+ )
+
+ $vstsCommandString = "vso[task.setvariable variable=$Name]$Value"
+ Write-Verbose -Verbose -Message ("sending " + $vstsCommandString)
+ Write-Host "##$vstsCommandString"
+
+ # also set in the current session
+ Set-Item -Path "env:$Name" -Value $Value
+}
diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj
index b3ed7b19be1..f0b5a2c5d4c 100644
--- a/test/tools/WebListener/WebListener.csproj
+++ b/test/tools/WebListener/WebListener.csproj
@@ -10,9 +10,4 @@
-
-
-
-
-
diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml
index 221a385b8b8..ab6ef4bc1aa 100644
--- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml
+++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml
@@ -16,12 +16,6 @@ steps:
displayName: Update PSGet and PackageManagement
condition: succeededOrFailed()
- - pwsh: |
- Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1"
- SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory)
- displayName: Download packages from PSGallery that need to be updated
- condition: succeededOrFailed()
-
- pwsh: |
Write-Verbose -Verbose "Packages to upload"
if(Test-Path $(Build.ArtifactStagingDirectory)) { Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg" | ForEach-Object { $_.FullName }}
diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1
deleted file mode 100644
index d0aeac9da54..00000000000
--- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1
+++ /dev/null
@@ -1,186 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-<#
-.SYNOPSIS
-Downloads to packages from PowerShell Gallery which are missing from the Azure DevOps Artifacts feed.
-
-.PARAMETER AzureDevOpsPAT
-PAT for the username used for authenticating to the Azure DevOps Artifacts feed.
-
-.PARAMETER Destination
-Path to the folder where the packages should be stored for uploading to Azure DevOps Artifacts feed.
-
-#>
-function SyncGalleryToAzArtifacts {
- param(
- [Parameter(Mandatory = $true)] [string] $AzDevOpsFeedUserName,
- [Parameter(Mandatory = $true)] [string] $AzDevOpsPAT,
- [Parameter(Mandatory = $true)] [string] $Destination
- )
-
- $csproj = [xml] (Get-Content 'src/Modules/PSGalleryModules.csproj')
- $packages = @($csproj.Project.ItemGroup.PackageReference | ForEach-Object { [ordered] @{Name = $_.Include; Version = $_.Version }})
-
- $galleryPackages = @()
- $azArtifactsPackages = @()
- $modulesToUpdate = @()
-
- $galleryUrl = 'https://www.powershellgallery.com/api/v2/'
- $azArtifactsUrl = 'https://mscodehub.pkgs.visualstudio.com/_packaging/pscore-release/nuget/v2'
-
- $azDevOpsCreds = [pscredential]::new($AzDevOpsFeedUserName, (ConvertTo-SecureString -String $AzDevOpsPAT -AsPlainText -Force))
-
- foreach ($package in $packages) {
- try {
- # Get module from gallery
- $foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $package.Name -Force -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
- Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnGallery.Version) in gallery"
- $galleryPackages += $foundPackageOnGallery
- } catch {
- if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
- # Log and ignore failure is required version is not found on gallery.
- Write-Warning "Module not found on gallery $($package.Name) - $($package.Version)"
- }
- else {
- Write-Error $_
- }
- }
-
- try {
- # Get module from Az Artifacts
- # There seems to be a bug in the feed with RequiredVersion matching. Adding workaround with post filtering.
- # Issue: https://github.com/OneGet/oneget/issues/397
- $foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $package.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
- Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnAz.Version) in azArtifacts"
- $azArtifactsPackages += $foundPackageOnAz
- } catch {
- if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
- # Log and add the module to update list.
- Write-Verbose -Verbose "Az Artifacts Module needs update to - $($package.Name) - $($package.Version)"
- $modulesToUpdate += $package
- }
- else {
- Write-Error $_
- }
- }
-
- # Check if Az package version is less that gallery version
- $pkgOnAzVersion = [semver]::new($foundPackageOnAz.Version)
- $pkgOnGalleryVersion = [semver]::new($foundPackageOnGallery.Version)
-
- if ($pkgOnAzVersion -lt $pkgOnGalleryVersion) {
- Write-Verbose -Verbose "Module needs to be updated $($package.Name) - $($foundPackageOnGallery.Version)"
- $modulesToUpdate += $foundPackageOnGallery
- } elseif ($pkgOnGalleryVersion -lt $pkgOnAzVersion) {
- Write-Warning "Newer version found on Az Artifacts - $($foundPackageOnAz.Name) - $($foundPackageOnAz.Version)"
- } else {
- Write-Verbose -Verbose "Module is in sync - $($package.Name)"
- }
- }
-
- "`nGallery Packages:"
- $galleryPackages
-
- "`nAz Artifacts Packages:`n"
- $azArtifactsPackages
-
- "`nModules to update:`n"
- $modulesToUpdate
-
- foreach ($package in $modulesToUpdate) {
- Write-Verbose -Verbose "Saving package $($package.Name) - $($package.Version)"
- Save-Package -Provider NuGet -Source $galleryUrl -Name $package.Name -RequiredVersion $package.Version -Path $Destination
- }
-
- if ($modulesToUpdate.Length -gt 0)
- {
- # Remove dependent packages downloaded by Save-Package if there are already present in AzArtifacts feed.
- try {
- $null = Register-PackageSource -Name local -Location $Destination -ProviderName NuGet -Force
- $packageNamesToKeep = @()
- $savedPackages = Find-Package -Source local -AllVersions -AllowPreReleaseVersion
-
- Write-Verbose -Verbose "Saved packages:"
- $savedPackages | Out-String | Write-Verbose -Verbose
-
- foreach($package in $savedPackages) {
- $pkgVersion = NormalizeVersion -version $package.Version
- $foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $package.Name -and (NormalizeVersion -version $_.Version) -eq $pkgVersion }
-
- if(-not $foundMatch) {
- Write-Verbose "Keeping package $($package.PackageFileName)" -Verbose
- $packageNamesToKeep += "{0}*.nupkg" -f $package.Name
- }
- }
-
- if ($packageNamesToKeep.Length -gt 0) {
- ## Removing only if we do have some packages to keep,
- ## otherwise the '$Destination' folder will be removed.
- Remove-Item -Path $Destination -Exclude $packageNamesToKeep -Recurse -Force -Verbose
- }
-
- Write-Verbose -Verbose "Packages kept for upload"
- Get-ChildItem $Destination | Out-String | Write-Verbose -Verbose
- }
- finally {
- Unregister-PackageSource -Name local -Force -ErrorAction SilentlyContinue
- }
- }
-}
-
-Function SortPackage {
- param(
- [Parameter(ValueFromPipeline = $true)]
- [Microsoft.PackageManagement.Packaging.SoftwareIdentity[]]
- $packages
- )
-
- Begin {
- $allPackages = @()
- }
-
- Process {
- $allPackages += $packages
- }
-
- End {
- $versions = $allPackages.Version |
- ForEach-Object { ($_ -split '-')[0] } |
- Select-Object -Unique |
- Sort-Object -Descending -Property Version
-
- foreach ($version in $versions) {
- $exactMatch = $allPackages | Where-Object {
- Write-Verbose "testing $($_.version) -eq $version"
- $_.version -eq $version
- }
-
- if ($exactMatch) {
- Write-Output $exactMatch
- }
-
- $allPackages | Where-Object {
- $_.version -like "${version}-*"
- } | Sort-Object -Descending -Property Version | Write-Output
- }
- }
-}
-
-
-function NormalizeVersion {
- param ([string] $version)
-
- $sVer = if ($version -match "(\d+.\d+.\d+).0") {
- $Matches[1]
- } elseif ($version -match "^\d+.\d+$") {
- # Two digit versions are stored as three digit versions
- "$version.0"
- } else {
- $version
- }
-
- $sVer
-}
-
-Export-ModuleMember -Function 'SyncGalleryToAzArtifacts', 'SortPackage'
diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml
index c1c453bb145..22325661e65 100644
--- a/tools/releaseBuild/azureDevOps/releasePipeline.yml
+++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml
@@ -358,7 +358,8 @@ stages:
variables:
- group: 'Azure Blob variable group'
- - group: 'AzDevOpsArtifacts'
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
- group: ReleasePipelineSecrets
steps:
- template: templates/release-CreateGitHubDraft.yml
@@ -400,9 +401,10 @@ stages:
- ImageOverride -equals PSMMSUbuntu20.04-Secure
variables:
- - group: 'AzDevOpsArtifacts'
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
- group: 'packages.microsoft.com'
- - group: 'mscodehub-feed-read-akv'
+ - group: 'mscodehub-code-read-akv'
steps:
- template: templates/release-PublishPackageMsftCom.yml
parameters:
diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml b/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml
index 480658cc1cc..3c3d78637fe 100644
--- a/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml
+++ b/tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml
@@ -19,7 +19,8 @@ jobs:
- group: DotNetPrivateBuildAccess
- group: Azure Blob variable group
- group: ReleasePipelineSecrets
- - group: AzDevOpsArtifacts
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
pool:
name: PowerShell1ES
diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
index e72e3af9f9f..6f4a13b0418 100644
--- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
+++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
@@ -1,38 +1,7 @@
parameters:
- - name: "repoRoot"
- default: $(REPOROOT)
-
+- name: "repoRoot"
+ default: $(REPOROOT)
steps:
-- pwsh: |
- $configPath = "${env:NugetConfigDir}/nuget.config"
- Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
- New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
-
- if(-not (Test-Path $configPath))
- {
- throw "nuget.config is not created"
- }
- Get-Content $configPath | Write-Verbose -Verbose
- displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
- condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
- env:
- NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
-
-- pwsh: |
- $configPath = "${env:NugetConfigDir}/nuget.config"
- Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
- New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
-
- if(-not (Test-Path $configPath))
- {
- throw "nuget.config is not created"
- }
- Get-Content $configPath | Write-Verbose -Verbose
- displayName: 'Add nuget.config for Azure DevOps feed for packages'
- condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
- env:
- NugetConfigDir: ${{ parameters.repoRoot }}
-
-- task: nuget-security-analysis@0
- displayName: 'Run Secure Supply Chain analysis'
-
+ - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
+ parameters:
+ repoRoot: $(REPOROOT)
diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml
index 776382c952d..f22aaf9e5f8 100644
--- a/tools/releaseBuild/azureDevOps/templates/mac.yml
+++ b/tools/releaseBuild/azureDevOps/templates/mac.yml
@@ -47,7 +47,7 @@ jobs:
repoRoot: $(PowerShellRoot)
- pwsh: |
- $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
+ $env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
$(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks
diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml
index b57ed223cae..479048fc19b 100644
--- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml
+++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml
@@ -20,6 +20,10 @@ parameters:
steps:
+- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
+ parameters:
+ repoRoot: $(REPOROOT)
+
- pwsh: |
Import-Module "$env:REPOROOT/build.psm1" -Force
Start-PSBootstrap
diff --git a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml b/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml
index 7039b64b7ed..861cf48c35a 100644
--- a/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml
@@ -19,7 +19,7 @@ steps:
"--verbose",
"--branch",
"$branch",
- "https://$(mscodehubPackageReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
+ "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
'$(Pipeline.Workspace)/tools'
$gitArgs | Write-Verbose -Verbose
git $gitArgs
diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml
index fc89e6966f0..5064e063e18 100644
--- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml
@@ -10,7 +10,9 @@ jobs:
# testing
vmImage: ${{ parameters.imageName }}
variables:
- - group: AzDevOpsArtifacts
+ - group: mscodehub-feed-read-general
+ - group: mscodehub-feed-read-akv
+ - group: DotNetPrivateBuildAccess
steps:
- checkout: self
clean: true
@@ -38,11 +40,13 @@ jobs:
runBranch: '$(Build.SourceBranch)'
artifact: metadata
path: '$(Pipeline.Workspace)/releasePipeline/metadata'
- allowPartiallySucceededBuilds: true
+
+ - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
+ parameters:
+ repoRoot: $(Build.SourcesDirectory)
- pwsh: |
Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force
- New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination '$(Build.SourcesDirectory)/test/hosting'
Write-Verbose -Verbose "Capture hosting folder files"
Get-ChildItem '$(Build.SourcesDirectory)/test/hosting'