Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4488,6 +4488,64 @@ $dStopped = Get-HarnessDuration $byN[2]; $dWaiting = Get-HarnessDuration $byN[3]
Test-Assert ($byN.Count -eq 4 -and $byN[2].Aborted -and $dStopped -is [double] -and $dStopped -gt 0 -and $dStopped -le 5 -and $byN[3].Aborted -and $null -eq $dWaiting -and $null -eq $dFailed) ("an abort: the force-stopped program has the time it ran, one never started and one that couldn't start have `$null (" + $dStopped + ", " + $dWaiting + ", " + $dFailed + ")")
Write-Host ""

Write-Host "=== AZ: programs left holding their output don't hold up a new program's ===" -ForegroundColor Cyan
# Each holder is a ping, started by a cmd that exits at once, holding stdout
# and stderr: two blocked reads. Enough of them to take more threads than the
# pool's minimum. With the reads on pool threads, the new program's reads
# queued behind the holders' while the pool grew a thread at a time, and it
# came back empty at its stream timeout. In a fresh PowerShell: earlier
# sections raise this one's pool minimum (the 20,000-job dispatch to over
# 2000), and once raised, lowering it again doesn't bring back the slow growth.
Reset-Harness
$azScript = Join-Path $env:TEMP ("pk-heldreads-" + [guid]::NewGuid().ToString("N") + ".ps1")
[System.IO.File]::WriteAllText($azScript, @'
param([string]$Kit)
Set-StrictMode -Version Latest
. $Kit
$null = Initialize-ParallelKit
$minWorker = 0; $minIo = 0
[System.Threading.ThreadPool]::GetMinThreads([ref]$minWorker, [ref]$minIo)
$count = [int][Math]::Ceiling($minWorker / 2) + 20
$cmd = Join-Path $env:SystemRoot 'System32\cmd.exe'
$held = [System.Collections.Generic.List[hashtable]]::new()
try {
for ($i = 0; $i -lt $count; $i++) { $held.Add((Start-PkProcess -FilePath $cmd -Arguments '/d /c start /b ping -n 91 127.0.0.1 >nul')) }
foreach ($s in $held) { $null = $s['Proc'].WaitForExit(10000) }
$sw = [System.Diagnostics.Stopwatch]::StartNew()
$r = Invoke-PkProcessWithTimeout -FilePath $cmd -Arguments '/d /c echo plain' -StreamTimeoutMs 5000
$sw.Stop()
[Console]::Out.WriteLine('RESULT|' + ([string]$r.Stdout).Trim() + '|' + $r.StreamTimedOut + '|' + [math]::Round($sw.Elapsed.TotalSeconds, 2) + '|' + $count + '|' + $minWorker)
} finally {
# Only the pings this script's own cmds started, found by parent PID.
$parents = @{}; foreach ($s in $held) { $parents[[int]$s['Proc'].Id] = $true }
foreach ($p in @(Get-CimInstance Win32_Process -Filter "Name='PING.EXE'")) {
if ($parents.ContainsKey([int]$p.ParentProcessId)) { Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue }
}
$tasks = [System.Collections.Generic.List[System.Threading.Tasks.Task]]::new()
foreach ($s in $held) { $tasks.Add($s['StdoutTask']); $tasks.Add($s['StderrTask']) }
$drained = $false
try { $drained = [System.Threading.Tasks.Task]::WaitAll($tasks.ToArray(), 15000) } catch { $drained = $true } # a faulted read has still ended, which is all this waits for
[Console]::Out.WriteLine('DRAINED|' + $drained + '|' + $tasks.Count)
}
'@)
try {
$azRun = Invoke-PkProcessWithTimeout -FilePath $selfExe -Arguments ('-NoProfile -ExecutionPolicy Bypass -File "' + $azScript + '" -Kit "' + (Join-Path $PSScriptRoot 'ParallelKit.ps1') + '"') -TimeoutMs 120000
$azLines = @(([string]$azRun.Stdout) -split "`r?`n")
$azResult = @($azLines | Where-Object { $_ -like 'RESULT|*' })
$azDrain = @($azLines | Where-Object { $_ -like 'DRAINED|*' })
$azOk = $false; $azText = ([string]$azRun.Stdout + [string]$azRun.Stderr).Trim()
if ($azResult.Count -eq 1) {
$f = $azResult[0].Split('|')
$azOk = ($f[1] -eq 'plain' -and $f[2] -eq 'False' -and [double]$f[3] -lt 2.5)
$azText = $f[4] + " programs' output held open (pool minimum " + $f[5] + "): '" + $f[1] + "', StreamTimedOut=" + $f[2] + ", " + $f[3] + " s"
}
Test-Assert $azOk ("a new program still gets its output well inside its stream timeout while others' output is held open (" + $azText + ")")
Test-Assert ($azDrain.Count -eq 1 -and $azDrain[0] -like 'DRAINED|True|*') ("once the holders are killed, all their reads end (" + ($azDrain -join '') + ")")
} finally {
[System.IO.File]::Delete($azScript)
}
Write-Host ""

Set-PkMessageHandler -Handler $null
Reset-PkAbortState

Expand Down
21 changes: 12 additions & 9 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ public static class PkState

// Bump this, and the check after the type definitions, whenever any C#
// in this file changes: a session keeps the types it compiled first.
public const int TypesVersion = 7;
public const int TypesVersion = 8;

// Locked: Register-/Unregister-PkCancellationSource are callable from
// worker runspaces, and List<T> is not safe for concurrent writers.
Expand Down Expand Up @@ -250,9 +250,12 @@ public sealed class PkStartedProcess
// can hold the stream open long after it exits. The task's result is the
// whole text, at the end of the stream. The reader's decoder carries a
// character split across two reads over to the next, so a snapshot never
// ends in half a character. A read blocks a thread-pool thread, as
// ReadToEndAsync did; it ends when every holder of the stream's write end
// has let go (at the latest when PowerShell exits: they're in the Job Object).
// ends in half a character. Each read runs on a thread of its own
// (LongRunning), not a thread-pool thread: a read blocks until every holder
// of the stream's write end has let go (at the latest when PowerShell exits:
// they're in the Job Object), and leftover programs holding their output
// would otherwise use up the pool, so a new program's reads would wait for
// the pool to grow and hit their stream timeout.
public sealed class PkStreamReadTask : Task<string>
{
private sealed class Buffer
Expand All @@ -278,7 +281,7 @@ public sealed class PkStreamReadTask : Task<string>
}

private readonly Buffer _buffer;
private PkStreamReadTask(Buffer buffer) : base(new Func<string>(buffer.Pump)) { _buffer = buffer; }
private PkStreamReadTask(Buffer buffer) : base(new Func<string>(buffer.Pump), TaskCreationOptions.LongRunning) { _buffer = buffer; }

public static PkStreamReadTask Run(TextReader reader)
{
Expand Down Expand Up @@ -779,7 +782,7 @@ public static class PkChildProcessJob
# A session that loaded an older copy keeps its compiled types. Say so,
# rather than fail later with a missing-method error. Keep this number in
# sync with [PkState]::TypesVersion.
if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 7) {
if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 8) {
throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one."
}

Expand Down Expand Up @@ -934,9 +937,9 @@ function Initialize-PkChildProcessJob {
}

function Initialize-PkThreadPool {
# Each redirected child holds two pool threads for its output reads, and
# the pool grows slowly past its minimum, so a burst of processes could
# stall their reads. Raise the minimum up front.
# Raises the pool's minimum, since the pool grows slowly past it. This was
# for the output reads, two per program; those now run on threads of their
# own (PkStreamReadTask), so the kit's own work doesn't depend on it.
param([int]$MaxConcurrentProcesses = 0)
$clampedProcs = [Math]::Min([Math]::Max($MaxConcurrentProcesses, 0), 16000)
$minimumWorkerThreads = [Math]::Max(128, (2 * $clampedProcs) + 32)
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth
- `Aborted` is `$true` for a command an abort killed or kept from starting.
- `TimedOut` is `$true` for a program `-TimeoutSeconds` killed.
- `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`.
- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. `StreamError` means a read failed, and that stream comes back `$null`.
- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it. `StreamError` means a read failed, and that stream comes back `$null`.

`Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`.

Expand Down
Loading