diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 2d95b46..e024656 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4488,6 +4488,64 @@ $dStopped = Get-HarnessDuration $byN[2]; $dWaiting = Get-HarnessDuration $byN[3] Test-Assert ($byN.Count -eq 4 -and $byN[2].Aborted -and $dStopped -is [double] -and $dStopped -gt 0 -and $dStopped -le 5 -and $byN[3].Aborted -and $null -eq $dWaiting -and $null -eq $dFailed) ("an abort: the force-stopped program has the time it ran, one never started and one that couldn't start have `$null (" + $dStopped + ", " + $dWaiting + ", " + $dFailed + ")") Write-Host "" +Write-Host "=== AZ: programs left holding their output don't hold up a new program's ===" -ForegroundColor Cyan +# Each holder is a ping, started by a cmd that exits at once, holding stdout +# and stderr: two blocked reads. Enough of them to take more threads than the +# pool's minimum. With the reads on pool threads, the new program's reads +# queued behind the holders' while the pool grew a thread at a time, and it +# came back empty at its stream timeout. In a fresh PowerShell: earlier +# sections raise this one's pool minimum (the 20,000-job dispatch to over +# 2000), and once raised, lowering it again doesn't bring back the slow growth. +Reset-Harness +$azScript = Join-Path $env:TEMP ("pk-heldreads-" + [guid]::NewGuid().ToString("N") + ".ps1") +[System.IO.File]::WriteAllText($azScript, @' +param([string]$Kit) +Set-StrictMode -Version Latest +. $Kit +$null = Initialize-ParallelKit +$minWorker = 0; $minIo = 0 +[System.Threading.ThreadPool]::GetMinThreads([ref]$minWorker, [ref]$minIo) +$count = [int][Math]::Ceiling($minWorker / 2) + 20 +$cmd = Join-Path $env:SystemRoot 'System32\cmd.exe' +$held = [System.Collections.Generic.List[hashtable]]::new() +try { + for ($i = 0; $i -lt $count; $i++) { $held.Add((Start-PkProcess -FilePath $cmd -Arguments '/d /c start /b ping -n 91 127.0.0.1 >nul')) } + foreach ($s in $held) { $null = $s['Proc'].WaitForExit(10000) } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $r = Invoke-PkProcessWithTimeout -FilePath $cmd -Arguments '/d /c echo plain' -StreamTimeoutMs 5000 + $sw.Stop() + [Console]::Out.WriteLine('RESULT|' + ([string]$r.Stdout).Trim() + '|' + $r.StreamTimedOut + '|' + [math]::Round($sw.Elapsed.TotalSeconds, 2) + '|' + $count + '|' + $minWorker) +} finally { + # Only the pings this script's own cmds started, found by parent PID. + $parents = @{}; foreach ($s in $held) { $parents[[int]$s['Proc'].Id] = $true } + foreach ($p in @(Get-CimInstance Win32_Process -Filter "Name='PING.EXE'")) { + if ($parents.ContainsKey([int]$p.ParentProcessId)) { Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue } + } + $tasks = [System.Collections.Generic.List[System.Threading.Tasks.Task]]::new() + foreach ($s in $held) { $tasks.Add($s['StdoutTask']); $tasks.Add($s['StderrTask']) } + $drained = $false + try { $drained = [System.Threading.Tasks.Task]::WaitAll($tasks.ToArray(), 15000) } catch { $drained = $true } # a faulted read has still ended, which is all this waits for + [Console]::Out.WriteLine('DRAINED|' + $drained + '|' + $tasks.Count) +} +'@) +try { + $azRun = Invoke-PkProcessWithTimeout -FilePath $selfExe -Arguments ('-NoProfile -ExecutionPolicy Bypass -File "' + $azScript + '" -Kit "' + (Join-Path $PSScriptRoot 'ParallelKit.ps1') + '"') -TimeoutMs 120000 + $azLines = @(([string]$azRun.Stdout) -split "`r?`n") + $azResult = @($azLines | Where-Object { $_ -like 'RESULT|*' }) + $azDrain = @($azLines | Where-Object { $_ -like 'DRAINED|*' }) + $azOk = $false; $azText = ([string]$azRun.Stdout + [string]$azRun.Stderr).Trim() + if ($azResult.Count -eq 1) { + $f = $azResult[0].Split('|') + $azOk = ($f[1] -eq 'plain' -and $f[2] -eq 'False' -and [double]$f[3] -lt 2.5) + $azText = $f[4] + " programs' output held open (pool minimum " + $f[5] + "): '" + $f[1] + "', StreamTimedOut=" + $f[2] + ", " + $f[3] + " s" + } + Test-Assert $azOk ("a new program still gets its output well inside its stream timeout while others' output is held open (" + $azText + ")") + Test-Assert ($azDrain.Count -eq 1 -and $azDrain[0] -like 'DRAINED|True|*') ("once the holders are killed, all their reads end (" + ($azDrain -join '') + ")") +} finally { + [System.IO.File]::Delete($azScript) +} +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index ce49069..675310b 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -164,7 +164,7 @@ public static class PkState // Bump this, and the check after the type definitions, whenever any C# // in this file changes: a session keeps the types it compiled first. - public const int TypesVersion = 7; + public const int TypesVersion = 8; // Locked: Register-/Unregister-PkCancellationSource are callable from // worker runspaces, and List is not safe for concurrent writers. @@ -250,9 +250,12 @@ public sealed class PkStartedProcess // can hold the stream open long after it exits. The task's result is the // whole text, at the end of the stream. The reader's decoder carries a // character split across two reads over to the next, so a snapshot never -// ends in half a character. A read blocks a thread-pool thread, as -// ReadToEndAsync did; it ends when every holder of the stream's write end -// has let go (at the latest when PowerShell exits: they're in the Job Object). +// ends in half a character. Each read runs on a thread of its own +// (LongRunning), not a thread-pool thread: a read blocks until every holder +// of the stream's write end has let go (at the latest when PowerShell exits: +// they're in the Job Object), and leftover programs holding their output +// would otherwise use up the pool, so a new program's reads would wait for +// the pool to grow and hit their stream timeout. public sealed class PkStreamReadTask : Task { private sealed class Buffer @@ -278,7 +281,7 @@ public sealed class PkStreamReadTask : Task } private readonly Buffer _buffer; - private PkStreamReadTask(Buffer buffer) : base(new Func(buffer.Pump)) { _buffer = buffer; } + private PkStreamReadTask(Buffer buffer) : base(new Func(buffer.Pump), TaskCreationOptions.LongRunning) { _buffer = buffer; } public static PkStreamReadTask Run(TextReader reader) { @@ -779,7 +782,7 @@ public static class PkChildProcessJob # A session that loaded an older copy keeps its compiled types. Say so, # rather than fail later with a missing-method error. Keep this number in # sync with [PkState]::TypesVersion. -if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 7) { +if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 8) { throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one." } @@ -934,9 +937,9 @@ function Initialize-PkChildProcessJob { } function Initialize-PkThreadPool { - # Each redirected child holds two pool threads for its output reads, and - # the pool grows slowly past its minimum, so a burst of processes could - # stall their reads. Raise the minimum up front. + # Raises the pool's minimum, since the pool grows slowly past it. This was + # for the output reads, two per program; those now run on threads of their + # own (PkStreamReadTask), so the kit's own work doesn't depend on it. param([int]$MaxConcurrentProcesses = 0) $clampedProcs = [Math]::Min([Math]::Max($MaxConcurrentProcesses, 0), 16000) $minimumWorkerThreads = [Math]::Max(128, (2 * $clampedProcs) + 32) diff --git a/README.md b/README.md index 7f8911a..a867422 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. -- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. `StreamError` means a read failed, and that stream comes back `$null`. +- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it. `StreamError` means a read failed, and that stream comes back `$null`. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`.