Try to diagnose memory access violation in Windows tests (#35926)

Release Notes:

- N/A
This commit is contained in:
Cole Miller 2025-08-11 13:06:31 -04:00 committed by GitHub
parent 39dfd52d04
commit 76b95d4f67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -20,7 +20,168 @@ runs:
with:
node-version: "18"
- name: Configure crash dumps
shell: powershell
run: |
# Record the start time for this CI run
$runStartTime = Get-Date
$runStartTimeStr = $runStartTime.ToString("yyyy-MM-dd HH:mm:ss")
Write-Host "CI run started at: $runStartTimeStr"
# Save the timestamp for later use
echo "CI_RUN_START_TIME=$($runStartTime.Ticks)" >> $env:GITHUB_ENV
# Create crash dump directory in workspace (non-persistent)
$dumpPath = "$env:GITHUB_WORKSPACE\crash_dumps"
New-Item -ItemType Directory -Force -Path $dumpPath | Out-Null
Write-Host "Setting up crash dump detection..."
Write-Host "Workspace dump path: $dumpPath"
# Note: We're NOT modifying registry on stateful runners
# Instead, we'll check default Windows crash locations after tests
- name: Run tests
shell: powershell
working-directory: ${{ inputs.working-directory }}
run: cargo nextest run --workspace --no-fail-fast
run: |
$env:RUST_BACKTRACE = "full"
# Enable Windows debugging features
$env:_NT_SYMBOL_PATH = "srv*https://msdl.microsoft.com/download/symbols"
# .NET crash dump environment variables (ephemeral)
$env:COMPlus_DbgEnableMiniDump = "1"
$env:COMPlus_DbgMiniDumpType = "4"
$env:COMPlus_CreateDumpDiagnostics = "1"
cargo nextest run --workspace --no-fail-fast
continue-on-error: true
- name: Analyze crash dumps
if: always()
shell: powershell
run: |
Write-Host "Checking for crash dumps..."
# Get the CI run start time from the environment
$runStartTime = [DateTime]::new([long]$env:CI_RUN_START_TIME)
Write-Host "Only analyzing dumps created after: $($runStartTime.ToString('yyyy-MM-dd HH:mm:ss'))"
# Check all possible crash dump locations
$searchPaths = @(
"$env:GITHUB_WORKSPACE\crash_dumps",
"$env:LOCALAPPDATA\CrashDumps",
"$env:TEMP",
"$env:GITHUB_WORKSPACE",
"$env:USERPROFILE\AppData\Local\CrashDumps",
"C:\Windows\System32\config\systemprofile\AppData\Local\CrashDumps"
)
$dumps = @()
foreach ($path in $searchPaths) {
if (Test-Path $path) {
Write-Host "Searching in: $path"
$found = Get-ChildItem "$path\*.dmp" -ErrorAction SilentlyContinue | Where-Object {
$_.CreationTime -gt $runStartTime
}
if ($found) {
$dumps += $found
Write-Host " Found $($found.Count) dump(s) from this CI run"
}
}
}
if ($dumps) {
Write-Host "Found $($dumps.Count) crash dump(s)"
# Install debugging tools if not present
$cdbPath = "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
if (-not (Test-Path $cdbPath)) {
Write-Host "Installing Windows Debugging Tools..."
$url = "https://go.microsoft.com/fwlink/?linkid=2237387"
Invoke-WebRequest -Uri $url -OutFile winsdksetup.exe
Start-Process -Wait winsdksetup.exe -ArgumentList "/features OptionId.WindowsDesktopDebuggers /quiet"
}
foreach ($dump in $dumps) {
Write-Host "`n=================================="
Write-Host "Analyzing crash dump: $($dump.Name)"
Write-Host "Size: $([math]::Round($dump.Length / 1MB, 2)) MB"
Write-Host "Time: $($dump.CreationTime)"
Write-Host "=================================="
# Set symbol path
$env:_NT_SYMBOL_PATH = "srv*C:\symbols*https://msdl.microsoft.com/download/symbols"
# Run analysis
$analysisOutput = & $cdbPath -z $dump.FullName -c "!analyze -v; ~*k; lm; q" 2>&1 | Out-String
# Extract key information
if ($analysisOutput -match "ExceptionCode:\s*([\w]+)") {
Write-Host "Exception Code: $($Matches[1])"
if ($Matches[1] -eq "c0000005") {
Write-Host "Exception Type: ACCESS VIOLATION"
}
}
if ($analysisOutput -match "EXCEPTION_RECORD:\s*(.+)") {
Write-Host "Exception Record: $($Matches[1])"
}
if ($analysisOutput -match "FAULTING_IP:\s*\n(.+)") {
Write-Host "Faulting Instruction: $($Matches[1])"
}
# Save full analysis
$analysisFile = "$($dump.FullName).analysis.txt"
$analysisOutput | Out-File -FilePath $analysisFile
Write-Host "`nFull analysis saved to: $analysisFile"
# Print stack trace section
Write-Host "`n--- Stack Trace Preview ---"
$stackSection = $analysisOutput -split "STACK_TEXT:" | Select-Object -Last 1
$stackLines = $stackSection -split "`n" | Select-Object -First 20
$stackLines | ForEach-Object { Write-Host $_ }
Write-Host "--- End Stack Trace Preview ---"
}
Write-Host "`n⚠ Crash dumps detected! Download the 'crash-dumps' artifact for detailed analysis."
# Copy dumps to workspace for artifact upload
$artifactPath = "$env:GITHUB_WORKSPACE\crash_dumps_collected"
New-Item -ItemType Directory -Force -Path $artifactPath | Out-Null
foreach ($dump in $dumps) {
$destName = "$($dump.Directory.Name)_$($dump.Name)"
Copy-Item $dump.FullName -Destination "$artifactPath\$destName"
if (Test-Path "$($dump.FullName).analysis.txt") {
Copy-Item "$($dump.FullName).analysis.txt" -Destination "$artifactPath\$destName.analysis.txt"
}
}
Write-Host "Copied $($dumps.Count) dump(s) to artifact directory"
} else {
Write-Host "No crash dumps from this CI run found"
}
- name: Upload crash dumps
if: always()
uses: actions/upload-artifact@v4
with:
name: crash-dumps-${{ github.run_id }}-${{ github.run_attempt }}
path: |
crash_dumps_collected/*.dmp
crash_dumps_collected/*.txt
if-no-files-found: ignore
retention-days: 7
- name: Check test results
shell: powershell
working-directory: ${{ inputs.working-directory }}
run: |
# Re-check test results to fail the job if tests failed
if ($LASTEXITCODE -ne 0) {
Write-Host "Tests failed with exit code: $LASTEXITCODE"
exit $LASTEXITCODE
}