Collect and upload kernel crash dumps. (#808)

microsoft · Mar 16, 2022 · fc2f504 · fc2f504
1 parent a8e3a30
commit fc2f504
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 63 deletions.
diff --git a/.github/workflows/driver_test_vm.yml b/.github/workflows/driver_test_vm.yml
@@ -109,18 +109,11 @@ jobs:
         ./execute_ebpf_cicd_tests.ps1
 
     - name: Clean up CI/CD Tests
+      if: always()
       working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
       run: |
         ./cleanup_ebpf_cicd_tests.ps1
 
-    - name: Upload any crash dumps
-      uses: actions/upload-artifact@v2
-      if: failure()
-      with:
-        name: Crash-Dumps-${{env.BUILD_PLATFORM}}-${{env.BUILD_CONFIGURATION}}
-        path: c:/dumps/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
-        retention-days: 5
-
     - name: Upload log files
       if: always()
       uses: actions/upload-artifact@v2

diff --git a/docs/SelfHostedRunnerSetup.md b/docs/SelfHostedRunnerSetup.md
@@ -32,4 +32,10 @@ This document discusses the steps to set up such a selfhosted actions-runner tha
 9) Store the VM administrator credential:
    1) `Install-Module CredentialManager -force`
    2) `New-StoredCredential -Target `**`TEST_VM`**` -Username <VM Administrator> -Password <VM Administrator account password> -Persist LocalMachine`
-10) Reboot the runner.
+10) Install procdump on the two test VMs and configure as follows:
+    1) Download Procdump.zip from https://download.sysinternals.com/files/Procdump.zip
+    2) Extract the files into "C:\Program Files\ProcDump"
+    3) $env:Path += ";C:\Program Files\ProcDump"
+    4) Create two directories c:\dumps\x64\Release and c:\dumps\x64\Debug
+    5) Run: `procdump64.exe -accepteula -i -r -ma <path>` once for each of the two directories above, substituting `path` for the directory path.
+11) Reboot the runner.
diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1
@@ -174,19 +174,21 @@ function Export-BuildArtifactsToVMs
 
     foreach($VM in $VMList) {
         $VMName = $VM.Name
-        Write-Log "Exporting all files in $pwd to c:\eBPF\ on $VMName"
         $TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
         $VMSession = New-PSSession -VMName $VMName -Credential $TestCredential
         if (!$VMSession) {
             throw "Failed to create PowerShell session on $VMName."
         } else {
             Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
-                if(!(Test-Path "C:\eBPF")) {
-                    New-Item -ItemType Directory -Path "C:\eBPF"
+                if(!(Test-Path "$Env:SystemDrive\eBPF")) {
+                    New-Item -ItemType Directory -Path "$Env:SystemDrive\eBPF"
                 }
+                return $Env:SystemDrive
             }
+            $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}
         }
-        Copy-Item -ToSession $VMSession -Path "$pwd\*" -Exclude "*.pdb" -Destination "C:\eBPF\" -Recurse -Force 2>&1 -ErrorAction Stop | Write-Log
+        Write-Log "Exporting all files in $pwd to $VMSystemDrive\eBPF on $VMName"
+        Copy-Item -ToSession $VMSession -Path "$pwd\*" -Exclude "*.pdb" -Destination "$VMSystemDrive\eBPF" -Recurse -Force 2>&1 -ErrorAction Stop | Write-Log
 
         Write-Log "Export completed." -ForegroundColor Green
     }
@@ -200,31 +202,58 @@ function Import-ResultsFromVM
 {
     param([Parameter(Mandatory=$True)] $VMList)
 
+    # Wait for all VMs to be in ready state, in case the test run caused any VM to crash.
+    Wait-AllVMsToInitialize -VMList $VMList -UserName $Admin -AdminPassword $AdminPassword
+
     foreach($VM in $VMList) {
         $VMName = $VM.Name
         Write-Log "Importing TestLogs from $VMName"
+        if (!(Test-Path ".\TestLogs\$VMName")) {
+            New-Item -ItemType Directory -Path ".\TestLogs\$VMName"
+        }
+
         $TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
         $VMSession = New-PSSession -VMName $VMName -Credential $TestCredential
         if (!$VMSession) {
             throw "Failed to create PowerShell session on $VMName."
         }
-        if (!(Test-Path ".\TestLogs")) {
-            New-Item -ItemType Directory -Path ".\TestLogs"
+        $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}
+
+        # Copy kernel crash dumps if any.
+        if (!(Test-Path ".\TestLogs\$VMName\KernelDumps\")) {
+            New-Item -ItemType Directory -Path ".\TestLogs\$VMName\KernelDumps"
         }
+
+        Invoke-Command -Session $VMSession -ScriptBlock {
+            if (!(Test-Path "$Env:SystemDrive\KernelDumps")) {
+                New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps"
+            }
+            Move-Item $Env:WinDir\*.dmp $Env:SystemDrive\KernelDumps -ErrorAction Ignore
+        }
+        Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName\KernelDumps" -Force -ErrorAction Ignore 2>&1 | Write-Log
+
+        # Copy user mode crash dumps if any.
+        if (!(Test-Path ".\TestLogs\$VMName\UMDumps\")) {
+            New-Item -ItemType Directory -Path ".\TestLogs\$VMName\UMDumps"
+        }
+
+        Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps\x64" -Destination ".\TestLogs\$VMName\UMDumps" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+
         # Copy logs from Test VM.
-        Write-Log ("Copy {0}_{1} from C:\eBPF on test VM to $pwd\TestLogs" -f $VMName, $LogFileName)
-        Copy-Item -FromSession $VMSession ("C:\eBPF\{0}_{1}" -f $VMName, $LogFileName) -Destination ".\TestLogs" -Recurse -Force -ErrorAction Stop 2>&1 | Write-Log
+        if (!(Test-Path ".\TestLogs\$VMName\Logs")) {
+            New-Item -ItemType Directory -Path ".\TestLogs\$VMName\Logs"
+        }
 
-        # Move runner test logs to TestLogs folder.
-        Move-Item $LogFileName -Destination ".\TestLogs" -Force -ErrorAction Stop 2>&1 | Write-Log
+        Write-Log ("Copy $LogFileName from eBPF on $VMName to $pwd\TestLogs")
+        Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Stop 2>&1 | Write-Log
 
         # Copy ETL from Test VM.
         $EtlFile = $LogFileName.Substring(0, $LogFileName.IndexOf('.')) + ".etl"
-
-        Write-Log ("Copy {0}_{1} from C:\eBPF on test VM to $pwd\TestLogs" -f $VMName, $EtlFile)
-        Copy-Item -FromSession $VMSession ("C:\eBPF\{0}_{1}" -f $VMName, $EtlFile) -Destination ".\TestLogs" -Recurse -Force -ErrorAction Stop 2>&1 | Write-Log
-
+        Write-Log ("Copy $EtlFile from eBPF on $VMName to $pwd\TestLogs")
+        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Stop 2>&1 | Write-Log
     }
+    # Move runner test logs to TestLogs folder.
+    Move-Item $LogFileName -Destination ".\TestLogs" -Force -ErrorAction Stop 2>&1 | Write-Log
 }
 
 function Install-eBPFComponentsOnVM
@@ -237,10 +266,12 @@ function Install-eBPFComponentsOnVM
     Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
         param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
+        $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
         Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
         Import-Module $WorkingDirectory\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
         Install-eBPFComponents
-    } -ArgumentList ("C:\eBPF", ("{0}_{1}" -f $VMName, $LogFileName)) -ErrorAction Stop
+    } -ArgumentList ("eBPF", $LogFileName) -ErrorAction Stop
     Write-Log "eBPF components installed on $VMName" -ForegroundColor Green
 }
 
@@ -260,7 +291,7 @@ function Initialize-NetworkInterfacesOnVMs
             param([Parameter(Mandatory=$True)] $InterfaceList,
                   [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
                   [Parameter(Mandatory=$True)] [string] $LogFileName)
-
+            $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
             Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
 
             foreach ($Interface in $InterfaceList) {
@@ -277,6 +308,6 @@ function Initialize-NetworkInterfacesOnVMs
                 New-NetIPAddress -ifAlias "$InterfaceAlias" -IPAddress $V6Address -PrefixLength 64 -ErrorAction Stop | Out-Null
                 Write-Log "Address configured."
             }
-        } -ArgumentList ($Interfaces, "C:\eBPF", ("{0}_{1}" -f $VMName, $LogFileName)) -ErrorAction Stop
+        } -ArgumentList ($Interfaces, "eBPF", $LogFileName) -ErrorAction Stop
     }
 }
diff --git a/scripts/run_tests.psm1 b/scripts/run_tests.psm1
@@ -71,6 +71,8 @@ function Invoke-CICDTests
 {
     param([parameter(Mandatory=$true)][bool] $VerboseLogs)
 
+    pushd $WorkingDirectory
+
     try {
 
          $TestList = @(
@@ -90,6 +92,8 @@ function Invoke-CICDTests
         Write-Log "One or more tests failed."
         throw
     }
+
+    popd
 }
 
 function Invoke-XDPTest
@@ -118,4 +122,6 @@ function Invoke-XDPTest
     Write-Log "$XDPTestName Test Passed" -ForegroundColor Green
 
     popd
-}
+}
+
+Pop-Location
diff --git a/scripts/setup_ebpf_cicd_tests.ps1 b/scripts/setup_ebpf_cicd_tests.ps1
@@ -27,7 +27,7 @@ $MultiVMTestConfig = $TestExecutionConfig.MultiVMTest
 Remove-Item "$PSScriptRoot\$LogFileName" -ErrorAction SilentlyContinue
 foreach($VM in $VMList) {
     $VMName = $VM.Name
-    Remove-Item ("$PSScriptRoot\{0}_{1}" -f $VMName, $LogFileName) -ErrorAction SilentlyContinue
+    Remove-Item $PSScriptRoot\$LogFileName -ErrorAction SilentlyContinue
 }
 Remove-Item ".\TestLogs" -Recurse -Confirm:$false -ErrorAction SilentlyContinue
 

diff --git a/scripts/vm_run_tests.psm1 b/scripts/vm_run_tests.psm1
@@ -25,10 +25,12 @@ function Invoke-CICDTestsOnVM
         param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName,
               [Parameter(Mandatory=$True)] [bool] $VerboseLogs)
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
         Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
         Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
         Invoke-CICDTests -VerboseLogs $VerboseLogs 2>&1 | Write-Log
-    } -ArgumentList ("C:\eBPF", ("{0}_{1}" -f $VMName, $LogFileName), $VerboseLogs) -ErrorAction Stop
+    } -ArgumentList ("eBPF", $LogFileName, $VerboseLogs) -ErrorAction Stop
 }
 
 function Add-eBPFProgramOnVM
@@ -47,19 +49,20 @@ function Add-eBPFProgramOnVM
               [string] $Interface,
               [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
-            Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
-            Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
-
-            if ([System.String]::IsNullOrEmpty($Interface)){
-                Write-Log "Loading $Program on $VM."
-                $ProgId = Invoke-NetshEbpfCommand -Arguments "add program $WorkingDirectory\$Program"
-            } else {
-                Write-Log "Loading $Program on interface $Interface on $VM."
-                $ProgId = Invoke-NetshEbpfCommand -Arguments "add program $WorkingDirectory\$Program interface=""$Interface"""
-            }
-            Write-Log "Loaded $Program with $ProgId" -ForegroundColor Green
-            return $ProgId
-    } -ArgumentList ($VM, $Program, $Interface, "C:\eBPF", ("{0}_{1}" -f $VM, $LogFileName)) -ErrorAction Stop
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
+        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+        Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
+        if ([System.String]::IsNullOrEmpty($Interface)){
+            Write-Log "Loading $Program on $VM."
+            $ProgId = Invoke-NetshEbpfCommand -Arguments "add program $WorkingDirectory\$Program"
+        } else {
+            Write-Log "Loading $Program on interface $Interface on $VM."
+            $ProgId = Invoke-NetshEbpfCommand -Arguments "add program $WorkingDirectory\$Program interface=""$Interface"""
+        }
+        Write-Log "Loaded $Program with $ProgId" -ForegroundColor Green
+        return $ProgId
+    } -ArgumentList ($VM, $Program, $Interface, "eBPF", $LogFileName) -ErrorAction Stop
 
     return $ProgId
 }
@@ -80,11 +83,13 @@ function Set-eBPFProgramOnVM
               [parameter(Mandatory=$true)] [string] $Interface,
               [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
-            Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
-            Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
-            Write-Log "Setting program $ProgId at interface $Interface on $VM."
-            Invoke-NetshEbpfCommand -Arguments "set program $ProgId xdp interface=""$Interface"""
-    } -ArgumentList ($VM, $ProgId, $Interface, "C:\eBPF", ("{0}_{1}" -f $VM, $LogFileName)) -ErrorAction Stop
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
+        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+        Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
+        Write-Log "Setting program $ProgId at interface $Interface on $VM."
+        Invoke-NetshEbpfCommand -Arguments "set program $ProgId xdp interface=""$Interface"""
+    } -ArgumentList ($VM, $ProgId, $Interface, "eBPF", $LogFileName) -ErrorAction Stop
 }
 function Remove-eBPFProgramFromVM
 {
@@ -100,12 +105,14 @@ function Remove-eBPFProgramFromVM
               [Parameter(Mandatory=$True)] $ProgId,
               [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
-            Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
-            Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
-            Write-Log "Unloading program $ProgId from $VM."
-            Invoke-NetshEbpfCommand -Arguments "del program $ProgId"
-            return $ProgId
-    } -ArgumentList ($VM, $ProgId, "C:\eBPF", ("{0}_{1}" -f $VM, $LogFileName)) -ErrorAction Stop
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
+        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+        Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
+        Write-Log "Unloading program $ProgId from $VM."
+        Invoke-NetshEbpfCommand -Arguments "del program $ProgId"
+        return $ProgId
+    } -ArgumentList ($VM, $ProgId, "eBPF", $LogFileName) -ErrorAction Stop
 }
 
 function Invoke-XDPTestOnVM
@@ -125,11 +132,14 @@ function Invoke-XDPTestOnVM
               [Parameter(Mandatory=$True)] [string] $RemoteIPV6Address,
               [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
-            Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
-            Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
-            Write-Log "Invoking $XDPTestName on $VM"
-            Invoke-XDPTest $RemoteIPV4Address $RemoteIPV6Address $XDPTestName $WorkingDirectory
-    } -ArgumentList ($VM, $XDPTestName, $RemoteIPV4Address, $RemoteIPV6Address, "C:\eBPF", ("{0}_{1}" -f $VM, $LogFileName)) -ErrorAction Stop
+
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
+        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+        Import-Module $WorkingDirectory\run_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
+        Write-Log "Invoking $XDPTestName on $VM"
+        Invoke-XDPTest $RemoteIPV4Address $RemoteIPV6Address $XDPTestName $WorkingDirectory
+    } -ArgumentList ($VM, $XDPTestName, $RemoteIPV4Address, $RemoteIPV6Address, "eBPF", $LogFileName) -ErrorAction Stop
 }
 
 function Add-XDPTestFirewallRuleOnVM {
@@ -143,10 +153,12 @@ function Add-XDPTestFirewallRuleOnVM {
         param([Parameter(Mandatory=$True)] [string] $VM,
               [Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
-            Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
-            Write-Log "Allowing XDP test app through firewall on $VM."
-            New-NetFirewallRule -DisplayName "XDP_Test" -Program "$WorkingDirectory\xdp_tests.exe" -Direction Inbound -Action Allow
-    } -ArgumentList ($VM, "C:\eBPF", ("{0}_{1}" -f $VM, $LogFileName)) -ErrorAction Stop
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
+        Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+
+        Write-Log "Allowing XDP test app through firewall on $VM."
+        New-NetFirewallRule -DisplayName "XDP_Test" -Program "$WorkingDirectory\xdp_tests.exe" -Direction Inbound -Action Allow
+    } -ArgumentList ($VM, "eBPF", $LogFileName) -ErrorAction Stop
 }
 
 function Invoke-XDPTest1
@@ -299,10 +311,12 @@ function Stop-eBPFComponentsOnVM
     Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
         param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName)
+        $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
         Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
         Import-Module $WorkingDirectory\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
+
         Stop-eBPFComponents
-    } -ArgumentList ("C:\eBPF", ("{0}_{1}" -f $VMName, $LogFileName)) -ErrorAction Stop
+    } -ArgumentList ("eBPF", $LogFileName) -ErrorAction Stop
 }
 
 Pop-Location