Skip to content

Commit

Permalink
Update dependencies from https://github.com/dotnet/arcade build 20240…
Browse files Browse the repository at this point in the history
…531.1 (#2518)

[main] Update dependencies from dotnet/arcade
  • Loading branch information
dotnet-maestro[bot] authored Jun 3, 2024
1 parent 1ae9fc7 commit a399fa2
Show file tree
Hide file tree
Showing 22 changed files with 299 additions and 139 deletions.
12 changes: 6 additions & 6 deletions eng/Version.Details.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24278.1">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24281.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
<Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24278.1">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.24281.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
<Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="9.0.0-beta.24278.1">
<Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="9.0.0-beta.24281.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>0c4e52c37a29b75f64646220e0cc237177ce23a2</Sha>
<Sha>e6b3f32f9855dccbe2447471c8f729b66f17d242</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@
<SystemCollectionsImmutableVersion>8.0.0</SystemCollectionsImmutableVersion>
<SystemIOHashingVersion>8.0.0</SystemIOHashingVersion>
<MicrosoftBclTimeProviderVersion>8.0.0</MicrosoftBclTimeProviderVersion>
<MicrosoftDotNetXUnitExtensionsPackageVersion>9.0.0-beta.24278.1</MicrosoftDotNetXUnitExtensionsPackageVersion>
<MicrosoftDotNetXUnitExtensionsPackageVersion>9.0.0-beta.24281.1</MicrosoftDotNetXUnitExtensionsPackageVersion>
</PropertyGroup>
</Project>
159 changes: 71 additions & 88 deletions eng/common/SetupNugetSources.ps1
Original file line number Diff line number Diff line change
@@ -1,31 +1,32 @@
# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential
# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See
# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details
#
# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential
# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
#
# This script needs to be called in every job that will restore packages and which the base repo has
# private AzDO feeds in the NuGet.config.
#
# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
# from the AzureDevOps-Artifact-Feeds-Pats variable group.
#
# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing
# See example call for this script below.
#
# - task: PowerShell@2
# displayName: Setup Private Feeds Credentials
# displayName: Setup Internal Feeds
# condition: eq(variables['Agent.OS'], 'Windows_NT')
# inputs:
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
# - task: NuGetAuthenticate@1
#
# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
# This ensures that:
# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
# - The credential provider is installed
#
# This logic is also abstracted into enable-internal-sources.yml.

[CmdletBinding()]
param (
[Parameter(Mandatory = $true)][string]$ConfigFile,
[Parameter(Mandatory = $true)][string]$Password
[string]$Password
)

$ErrorActionPreference = "Stop"
Expand All @@ -34,12 +35,23 @@ Set-StrictMode -Version 2.0

. $PSScriptRoot\tools.ps1

$feedEndpoints = $null

# If a credential is provided, ensure that we don't overwrite the current set of
# credentials that may have been provided by a previous call to the credential provider.
if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) {
$feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json
} elseif ($Password) {
$feedEndpoints = @{ endpointCredentials = @() }
}

# Add source entry to PackageSources
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")

if ($packageSource -eq $null)
if ($null -eq $packageSource)
{
Write-Host "`tAdding package source" $SourceName
$packageSource = $doc.CreateElement("add")
$packageSource.SetAttribute("key", $SourceName)
$packageSource.SetAttribute("value", $SourceEndPoint)
Expand All @@ -48,58 +60,34 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
else {
Write-Host "Package source $SourceName already present."
}
AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
}

# Add a credential node for the specified source
function AddCredential($creds, $source, $username, $pwd) {
# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
{
$sourceElement = $doc.CreateElement($Source)
$creds.AppendChild($sourceElement) | Out-Null
}

# Add the <Username> node to the credential if none is found.
$usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
if ($usernameElement -eq $null)
{
$usernameElement = $doc.CreateElement("add")
$usernameElement.SetAttribute("key", "Username")
$sourceElement.AppendChild($usernameElement) | Out-Null
if ($pwd) {
$feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd
}
$usernameElement.SetAttribute("value", $Username)
}

# Add the <ClearTextPassword> to the credential if none is found.
# Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
# -> https://github.com/NuGet/Home/issues/5526
$passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
if ($passwordElement -eq $null)
{
$passwordElement = $doc.CreateElement("add")
$passwordElement.SetAttribute("key", "ClearTextPassword")
$sourceElement.AppendChild($passwordElement) | Out-Null
# Add a new feed endpoint credential
function AddCredential([array]$endpointCredentials, $source, $pwd) {
$endpointCredentials += @{
endpoint = $source;
password = $pwd
}

$passwordElement.SetAttribute("value", $pwd)
return $endpointCredentials
}

function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
function InsertMaestroInternalFeedCredentials($Sources, $pwd) {
$maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")

Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."

ForEach ($PackageSource in $maestroPrivateSources) {
Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
ForEach ($PackageSource in $maestroInternalSources) {
Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key
$feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd
}
}

function EnablePrivatePackageSources($DisabledPackageSources) {
$maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroPrivateSources) {
Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
function EnableInternalPackageSources($DisabledPackageSources) {
$maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroInternalSources) {
Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
# Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
}
Expand All @@ -110,58 +98,53 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) {
ExitWithExitCode 1
}

if (!$Password) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
ExitWithExitCode 1
}

# Load NuGet.config
$doc = New-Object System.Xml.XmlDocument
$filename = (Get-Item $ConfigFile).FullName
$doc.Load($filename)

# Get reference to <PackageSources> or create one if none exist already
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
if ($sources -eq $null) {
if ($null -eq $sources) {
$sources = $doc.CreateElement("packageSources")
$doc.DocumentElement.AppendChild($sources) | Out-Null
}

# Looks for a <PackageSourceCredentials> node. Create it if none is found.
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
if ($creds -eq $null) {
$creds = $doc.CreateElement("packageSourceCredentials")
$doc.DocumentElement.AppendChild($creds) | Out-Null
}

# Check for disabledPackageSources; we'll enable any darc-int ones we find there
$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
if ($disabledSources -ne $null) {
if ($null -ne $disabledSources) {
Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
EnablePrivatePackageSources -DisabledPackageSources $disabledSources
EnableInternalPackageSources -DisabledPackageSources $disabledSources
}

$userName = "dn-bot"

# Insert credential nodes for Maestro's private feeds
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
if ($Password) {
InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password
}

# 3.1 uses a different feed url format so it's handled differently here
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
if ($null -ne $dotnet31Source) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password
}

$dotnetVersions = @('5','6','7','8')

foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;
$dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
if ($dotnetSource -ne $null) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
if ($dotnetSource) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password
}
}

$doc.Save($filename)
$doc.Save($filename)

# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable
if ($null -ne $feedEndpoints) {
# ci is set to true so vso logging commands will be used.
$ci = $true
Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false
}
60 changes: 28 additions & 32 deletions eng/common/SetupNugetSources.sh
Original file line number Diff line number Diff line change
@@ -1,28 +1,27 @@
#!/usr/bin/env bash

# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
# This script adds internal feeds required to build commits that depend on intenral package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
#
# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential
# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
#
# This script needs to be called in every job that will restore packages and which the base repo has
# private AzDO feeds in the NuGet.config.
#
# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
# from the AzureDevOps-Artifact-Feeds-Pats variable group.
#
# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing.
# See example call for this script below.
#
# - task: Bash@3
# displayName: Setup Private Feeds Credentials
# displayName: Setup Internal Feeds
# inputs:
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
# arguments: $(Build.SourcesDirectory)/NuGet.config $Token
# arguments: $(Build.SourcesDirectory)/NuGet.config
# condition: ne(variables['Agent.OS'], 'Windows_NT')
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
# - task: NuGetAuthenticate@1
#
# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
# This ensures that:
# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt)
# - The credential provider is installed.
#
# This logic is also abstracted into enable-internal-sources.yml.

ConfigFile=$1
CredToken=$2
Expand All @@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then
ExitWithExitCode 1
fi

if [ -z "$CredToken" ]; then
Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
ExitWithExitCode 1
fi

if [[ `uname -s` == "Darwin" ]]; then
NL=$'\\\n'
TB=''
Expand Down Expand Up @@ -140,18 +134,20 @@ PackageSources+="$IFS"
PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
IFS=$PrevIFS

for FeedName in ${PackageSources[@]} ; do
# Check if there is no existing credential for this FeedName
grep -i "<$FeedName>" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding credentials for $FeedName."
if [ "$CredToken" ]; then
for FeedName in ${PackageSources[@]} ; do
# Check if there is no existing credential for this FeedName
grep -i "<$FeedName>" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding credentials for $FeedName."

PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"

sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
fi
done
sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
fi
done
fi

# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
grep -i "<disabledPackageSources>" $ConfigFile
Expand Down
13 changes: 13 additions & 0 deletions eng/common/core-templates/job/source-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ parameters:

is1ESPipeline: ''

# If set to true and running on a non-public project,
# Internal nuget and blob storage locations will be enabled.
# This is not enabled by default because many repositories do not need internal sources
# and do not need to have the required service connections approved in the pipeline.
enableInternalSources: false

jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
Expand Down Expand Up @@ -74,6 +80,13 @@ jobs:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
- 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error

- ${{ if eq(parameters.enableInternalSources, true) }}:
- template: /eng/common/core-templates/steps/enable-internal-sources.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- template: /eng/common/core-templates/steps/source-build.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
Expand Down
Loading

0 comments on commit a399fa2

Please sign in to comment.