Try to diagnose memory access violation in Windows tests (#35926)
Release Notes: - N/A
This commit is contained in:
parent
39dfd52d04
commit
76b95d4f67
1 changed files with 162 additions and 1 deletions
163
.github/actions/run_tests_windows/action.yml
vendored
163
.github/actions/run_tests_windows/action.yml
vendored
|
@ -20,7 +20,168 @@ runs:
|
|||
with:
|
||||
node-version: "18"
|
||||
|
||||
- name: Configure crash dumps
|
||||
shell: powershell
|
||||
run: |
|
||||
# Record the start time for this CI run
|
||||
$runStartTime = Get-Date
|
||||
$runStartTimeStr = $runStartTime.ToString("yyyy-MM-dd HH:mm:ss")
|
||||
Write-Host "CI run started at: $runStartTimeStr"
|
||||
|
||||
# Save the timestamp for later use
|
||||
echo "CI_RUN_START_TIME=$($runStartTime.Ticks)" >> $env:GITHUB_ENV
|
||||
|
||||
# Create crash dump directory in workspace (non-persistent)
|
||||
$dumpPath = "$env:GITHUB_WORKSPACE\crash_dumps"
|
||||
New-Item -ItemType Directory -Force -Path $dumpPath | Out-Null
|
||||
|
||||
Write-Host "Setting up crash dump detection..."
|
||||
Write-Host "Workspace dump path: $dumpPath"
|
||||
|
||||
# Note: We're NOT modifying registry on stateful runners
|
||||
# Instead, we'll check default Windows crash locations after tests
|
||||
|
||||
- name: Run tests
|
||||
shell: powershell
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: cargo nextest run --workspace --no-fail-fast
|
||||
run: |
|
||||
$env:RUST_BACKTRACE = "full"
|
||||
|
||||
# Enable Windows debugging features
|
||||
$env:_NT_SYMBOL_PATH = "srv*https://msdl.microsoft.com/download/symbols"
|
||||
|
||||
# .NET crash dump environment variables (ephemeral)
|
||||
$env:COMPlus_DbgEnableMiniDump = "1"
|
||||
$env:COMPlus_DbgMiniDumpType = "4"
|
||||
$env:COMPlus_CreateDumpDiagnostics = "1"
|
||||
|
||||
cargo nextest run --workspace --no-fail-fast
|
||||
continue-on-error: true
|
||||
|
||||
- name: Analyze crash dumps
|
||||
if: always()
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "Checking for crash dumps..."
|
||||
|
||||
# Get the CI run start time from the environment
|
||||
$runStartTime = [DateTime]::new([long]$env:CI_RUN_START_TIME)
|
||||
Write-Host "Only analyzing dumps created after: $($runStartTime.ToString('yyyy-MM-dd HH:mm:ss'))"
|
||||
|
||||
# Check all possible crash dump locations
|
||||
$searchPaths = @(
|
||||
"$env:GITHUB_WORKSPACE\crash_dumps",
|
||||
"$env:LOCALAPPDATA\CrashDumps",
|
||||
"$env:TEMP",
|
||||
"$env:GITHUB_WORKSPACE",
|
||||
"$env:USERPROFILE\AppData\Local\CrashDumps",
|
||||
"C:\Windows\System32\config\systemprofile\AppData\Local\CrashDumps"
|
||||
)
|
||||
|
||||
$dumps = @()
|
||||
foreach ($path in $searchPaths) {
|
||||
if (Test-Path $path) {
|
||||
Write-Host "Searching in: $path"
|
||||
$found = Get-ChildItem "$path\*.dmp" -ErrorAction SilentlyContinue | Where-Object {
|
||||
$_.CreationTime -gt $runStartTime
|
||||
}
|
||||
if ($found) {
|
||||
$dumps += $found
|
||||
Write-Host " Found $($found.Count) dump(s) from this CI run"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($dumps) {
|
||||
Write-Host "Found $($dumps.Count) crash dump(s)"
|
||||
|
||||
# Install debugging tools if not present
|
||||
$cdbPath = "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
|
||||
if (-not (Test-Path $cdbPath)) {
|
||||
Write-Host "Installing Windows Debugging Tools..."
|
||||
$url = "https://go.microsoft.com/fwlink/?linkid=2237387"
|
||||
Invoke-WebRequest -Uri $url -OutFile winsdksetup.exe
|
||||
Start-Process -Wait winsdksetup.exe -ArgumentList "/features OptionId.WindowsDesktopDebuggers /quiet"
|
||||
}
|
||||
|
||||
foreach ($dump in $dumps) {
|
||||
Write-Host "`n=================================="
|
||||
Write-Host "Analyzing crash dump: $($dump.Name)"
|
||||
Write-Host "Size: $([math]::Round($dump.Length / 1MB, 2)) MB"
|
||||
Write-Host "Time: $($dump.CreationTime)"
|
||||
Write-Host "=================================="
|
||||
|
||||
# Set symbol path
|
||||
$env:_NT_SYMBOL_PATH = "srv*C:\symbols*https://msdl.microsoft.com/download/symbols"
|
||||
|
||||
# Run analysis
|
||||
$analysisOutput = & $cdbPath -z $dump.FullName -c "!analyze -v; ~*k; lm; q" 2>&1 | Out-String
|
||||
|
||||
# Extract key information
|
||||
if ($analysisOutput -match "ExceptionCode:\s*([\w]+)") {
|
||||
Write-Host "Exception Code: $($Matches[1])"
|
||||
if ($Matches[1] -eq "c0000005") {
|
||||
Write-Host "Exception Type: ACCESS VIOLATION"
|
||||
}
|
||||
}
|
||||
|
||||
if ($analysisOutput -match "EXCEPTION_RECORD:\s*(.+)") {
|
||||
Write-Host "Exception Record: $($Matches[1])"
|
||||
}
|
||||
|
||||
if ($analysisOutput -match "FAULTING_IP:\s*\n(.+)") {
|
||||
Write-Host "Faulting Instruction: $($Matches[1])"
|
||||
}
|
||||
|
||||
# Save full analysis
|
||||
$analysisFile = "$($dump.FullName).analysis.txt"
|
||||
$analysisOutput | Out-File -FilePath $analysisFile
|
||||
Write-Host "`nFull analysis saved to: $analysisFile"
|
||||
|
||||
# Print stack trace section
|
||||
Write-Host "`n--- Stack Trace Preview ---"
|
||||
$stackSection = $analysisOutput -split "STACK_TEXT:" | Select-Object -Last 1
|
||||
$stackLines = $stackSection -split "`n" | Select-Object -First 20
|
||||
$stackLines | ForEach-Object { Write-Host $_ }
|
||||
Write-Host "--- End Stack Trace Preview ---"
|
||||
}
|
||||
|
||||
Write-Host "`n⚠️ Crash dumps detected! Download the 'crash-dumps' artifact for detailed analysis."
|
||||
|
||||
# Copy dumps to workspace for artifact upload
|
||||
$artifactPath = "$env:GITHUB_WORKSPACE\crash_dumps_collected"
|
||||
New-Item -ItemType Directory -Force -Path $artifactPath | Out-Null
|
||||
|
||||
foreach ($dump in $dumps) {
|
||||
$destName = "$($dump.Directory.Name)_$($dump.Name)"
|
||||
Copy-Item $dump.FullName -Destination "$artifactPath\$destName"
|
||||
if (Test-Path "$($dump.FullName).analysis.txt") {
|
||||
Copy-Item "$($dump.FullName).analysis.txt" -Destination "$artifactPath\$destName.analysis.txt"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Copied $($dumps.Count) dump(s) to artifact directory"
|
||||
} else {
|
||||
Write-Host "No crash dumps from this CI run found"
|
||||
}
|
||||
|
||||
- name: Upload crash dumps
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: crash-dumps-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
crash_dumps_collected/*.dmp
|
||||
crash_dumps_collected/*.txt
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
- name: Check test results
|
||||
shell: powershell
|
||||
working-directory: ${{ inputs.working-directory }}
|
||||
run: |
|
||||
# Re-check test results to fail the job if tests failed
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host "Tests failed with exit code: $LASTEXITCODE"
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue