Skip to content

Update Linux daily CI to run in a single agent & collect traces #12866

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TravisEz13 merged 4 commits into from
Jun 2, 2020
Merged

Update Linux daily CI to run in a single agent & collect traces #12866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d60947e
Collect CLR traces during Windows daily test run
daxian-dbw May 29, 2020
9e6f63e
Fix setting vsts variable
daxian-dbw May 29, 2020
536a62f
Change Linux daily CI to run tests in a single agent
daxian-dbw Jun 1, 2020
254c923
Capture the unzipped build directory
daxian-dbw Jun 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 157 additions & 0 deletions .vsts-ci/linux-daily.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr)
trigger:
# Batch merge builds together while a merge build is running
batch: true
branches:
include:
- master
- release*
- feature*
paths:
include:
- '*'
exclude:
- /.vsts-ci/misc-analysis.yml
- /.github/ISSUE_TEMPLATE/*
- /.dependabot/config.yml
pr:
branches:
include:
- master
- release*
- feature*
paths:
include:
- '*'
exclude:
- tools/releaseBuild/*
- tools/releaseBuild/azureDevOps/templates/*
- /.vsts-ci/misc-analysis.yml
- /.github/ISSUE_TEMPLATE/*
- /.dependabot/config.yml

variables:
DOTNET_CLI_TELEMETRY_OPTOUT: 1
POWERSHELL_TELEMETRY_OPTOUT: 1
# Avoid expensive initialization of dotnet cli, see: https://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
__SuppressAnsiEscapeSequences: 1

resources:
- repo: self
clean: true

stages:
- stage: BuildLinux
displayName: Build for Linux
jobs:
- template: templates/ci-build.yml
parameters:
pool: ubuntu-16.04
jobName: linux_build
displayName: linux Build

- stage: TestLinux
displayName: Test for Linux
jobs:
- job: linux_test
pool:
vmImage: ubuntu-16.04
displayName: Linux Test

steps:
- pwsh: |
Get-ChildItem -Path env:
displayName: Capture Environment
condition: succeededOrFailed()

- task: DownloadBuildArtifacts@0
displayName: 'Download Build Artifacts'
inputs:
downloadType: specific
itemPattern: |
build/**/*
xunit/**/*
downloadPath: '$(System.ArtifactsDirectory)'

- pwsh: |
Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse
displayName: 'Capture Artifacts Directory'
continueOnError: true

- pwsh: |
Import-Module .\tools\ci.psm1
Invoke-CIInstall -SkipUser
displayName: Bootstrap
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
$output = (Get-PSOptions).Output
$rootPath = Split-Path (Split-Path $output)
Expand-Archive -Path '$(System.ArtifactsDirectory)\build\build.zip' -DestinationPath $rootPath -Force

## Fix permissions
Get-ChildItem $rootPath -Recurse | ForEach-Object {
if ($_ -is [System.IO.DirectoryInfo]) {
chmod +rwx $_.FullName
} else {
chmod +rw $_.FullName
}
}
chmod a+x $output

Write-Host "=== Capture Unzipped Directory ==="
Get-ChildItem $rootPath -Recurse
displayName: 'Unzip Build and Fix Permissions'
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet CI
displayName: Test - UnelevatedPesterTests - CI
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet CI
displayName: Test - ElevatedPesterTests - CI
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others
displayName: Test - UnelevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others
displayName: Test - ElevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
$xUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\xUnitTestResults.xml"
Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile
displayName: Verify xUnit Test Results
condition: succeededOrFailed()

- stage: CodeCovTestPackage
displayName: CodeCoverage and Test Packages
dependsOn: [] # by specifying an empty array, this stage doesn't depend on the stage before it
jobs:
- job: CodeCovTestPackage
displayName: CodeCoverage and Test Packages
pool:
vmImage: ubuntu-16.04
steps:
- pwsh: |
Import-Module .\tools\ci.psm1
New-CodeCoverageAndTestPackage
displayName: CodeCoverage and Test Package
176 changes: 107 additions & 69 deletions .vsts-ci/windows-daily.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,75 +55,113 @@ stages:
displayName: Windows Test

steps:
- pwsh: |
Get-ChildItem -Path env:
displayName: 'Capture Environment'
condition: succeededOrFailed()

- task: DownloadBuildArtifacts@0
displayName: 'Download Build Artifacts'
inputs:
downloadType: specific
itemPattern: |
build/**/*
xunit/**/*
downloadPath: '$(System.ArtifactsDirectory)'

- pwsh: |
Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse
displayName: 'Capture Artifacts Directory'
continueOnError: true

# must be run frow Windows PowerShell
- powershell: |
Import-Module .\tools\ci.psm1
Invoke-CIInstall
displayName: Bootstrap
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
$path = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))
$rootPath = Split-Path -Path $path
Expand-Archive -Path '$(System.ArtifactsDirectory)\build\build.zip' -DestinationPath $rootPath -Force
displayName: 'Unzip Build'
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet CI
displayName: Test - UnelevatedPesterTests - CI
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet CI
displayName: Test - ElevatedPesterTests - CI
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others
displayName: Test - UnelevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others
displayName: Test - ElevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
$xUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\xUnitTestResults.xml"
Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile
displayName: Verify xUnit Test Results
condition: succeededOrFailed()
- pwsh: |
Get-ChildItem -Path env:
displayName: 'Capture Environment'
condition: succeededOrFailed()

- task: DownloadBuildArtifacts@0
displayName: 'Download Build Artifacts'
inputs:
downloadType: specific
itemPattern: |
build/**/*
xunit/**/*
downloadPath: '$(System.ArtifactsDirectory)'

- pwsh: |
Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse
displayName: 'Capture Artifacts Directory'
continueOnError: true

- pwsh: |
$capRootDir = Join-Path ([System.IO.Path]::GetTempPath()) "CAP"
$capUtilDir = Join-Path $capRootDir "Utils"

if (Test-Path $capRootDir) { Remove-Item $capRootDir -Recurse -Force }
New-Item $capUtilDir -ItemType Directory > $null

$capZipFile = Join-Path $capRootDir "cap.zip"
Invoke-WebRequest -Uri https://pscoretestdata.blob.core.windows.net/dotnet-cap/windows.zip -OutFile $capZipFile
Unblock-File -Path $capZipFile
Expand-Archive -Path $capZipFile -DestinationPath $capUtilDir -Force

Write-Host "=== Capture CAP Util Directory ==="
Get-ChildItem $capUtilDir -Recurse

Write-Host "##vso[task.setvariable variable=CapRootDir]$capRootDir"
Write-Host "##vso[task.setvariable variable=CapUtilDir]$capUtilDir"
displayName: 'Download CAP package'
condition: succeededOrFailed()

# must be run frow Windows PowerShell
- powershell: |
Import-Module .\tools\ci.psm1
Invoke-CIInstall
displayName: Bootstrap
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
$path = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))
$rootPath = Split-Path -Path $path
Expand-Archive -Path '$(System.ArtifactsDirectory)\build\build.zip' -DestinationPath $rootPath -Force
displayName: 'Unzip Build'
condition: succeeded()

- pwsh: |
Import-Module $(CapUtilDir)\CAPService.psm1
$dataDir = Start-TraceCollection -RootDir $(CapRootDir)
Write-Host "##vso[task.setvariable variable=CapDataDir]$dataDir"
displayName: 'Start CLR Trace Collection'
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet CI
displayName: Test - UnelevatedPesterTests - CI
condition: succeeded()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet CI
displayName: Test - ElevatedPesterTests - CI
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others
displayName: Test - UnelevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\tools\ci.psm1
Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json'
Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others
displayName: Test - ElevatedPesterTests - Others
condition: succeededOrFailed()

- pwsh: |
Import-Module .\build.psm1
$xUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\xUnitTestResults.xml"
Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile
displayName: Verify xUnit Test Results
condition: succeededOrFailed()

- pwsh: |
$capDataDir = '$(CapDataDir)'
$capModuleFile = '$(CapUtilDir)\CAPService.psm1'

if ((Test-Path $capModuleFile) -and (Test-Path $capDataDir)) {
Import-Module $capModuleFile
Stop-TraceCollection -DataDir $capDataDir -RepoRoot $pwd
}
displayName: 'Upload CLR Trace'
condition: always()

- stage: PackagingWin
displayName: Packaging for Windows
Expand Down