From 16828e546030a27981305419683b9f1b6a2ccf26 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:40:21 -0500 Subject: [PATCH 1/2] Reader cleanup: keep only the text handed over, report a failed read, close the reader - The first snapshot of a stream that hasn't ended is its text from then on. The read still drains to the end, so the program holding the pipe never blocks on it, but drops what it reads instead of building up text nobody can get. - Complete-PkProcess sets StreamError from a read that failed even when the other stream timed out: Task.WaitAll with a timeout only throws once every task has finished, so the failure went unreported. - The reader is closed when its stream ends, so a finished program's pipe handle goes then rather than at the next garbage collection. TypesVersion goes to 9: the C# changed. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 56 ++++++++++++++++++++++++++++++++ ParallelKit.ps1 | 72 ++++++++++++++++++++++++++++++++--------- README.md | 2 +- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index e024656..4da1d01 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4546,6 +4546,62 @@ try { } Write-Host "" +Write-Host "=== BA: after a stream timeout, the read drains without keeping the text ===" -ForegroundColor Cyan +# The read of a held stream goes on to its end, but nobody can get what it +# reads after the result was handed over, so it isn't kept. The reader is +# closed at the end of every stream, rather than at the next garbage +# collection. Test-only reads that a build without them lacks count as a +# failure, not an error. +Reset-Harness +function Get-HarnessReadState($Task, [string]$Name) { + try { return $Task.$Name } catch { return 'missing' } # an older build has no such test-only read +} +function Stop-HarnessPingsOf([int]$ParentId) { + foreach ($p in @(Get-CimInstance Win32_Process -Filter ("ParentProcessId=" + $ParentId + " AND Name='PING.EXE'"))) { Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue } +} +# A ping that writes a line a second, holding the output after cmd exits. +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo first& start /b ping -t 127.0.0.1' +try { + $null = $s['Proc'].WaitForExit(10000) + $r = Complete-PkProcess $s -StreamTimeoutMs 1500 + $baHeldAtResult = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength' + Start-Sleep -Seconds 3 + $baHeldLater = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength' + $baStillReading = -not $s['StdoutTask'].IsCompleted + $baLength = ([string]$r.Stdout).Length + Test-Assert ($r.StreamTimedOut -and $r.Stdout -match 'first' -and $baStillReading -and $baHeldAtResult -eq $baLength -and $baHeldLater -eq $baLength) ("a program that keeps writing after the result: the buffer stays at the result's " + $baLength + " characters (at the result " + $baHeldAtResult + ", 3 s later " + $baHeldLater + ", still reading " + $baStillReading + ")") +} finally { Stop-HarnessPingsOf -ParentId $s['Proc'].Id } +$baEnded = $s['StdoutTask'].Wait(10000) -and $s['StderrTask'].Wait(10000) +$baFinal = $null; if ($baEnded) { $baFinal = $s['StdoutTask'].Result } +Test-Assert ($baEnded -and $baFinal -eq $r.Stdout -and ([string]$baFinal).Length -eq $baLength) ("once the writer is killed the read ends, and its result is the text handed over, not what came after (" + $baEnded + ", " + ([string]$baFinal).Length + " characters)") +Test-Assert ((Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed') -eq $true) "a held stream's reader is closed when the stream ends" +$s['Proc'].Dispose() + +foreach ($baPath in 'created in the job', 'Process.Start fallback') { + [PkChildProcessJob]::TestUseProcessStart = ($baPath -eq 'Process.Start fallback') + try { + $s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo out& echo err 1>&2' + $r = Complete-PkProcess $s -Dispose + } finally { [PkChildProcessJob]::TestUseProcessStart = $false } + $baOut = Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed' + $baErr = Get-HarnessReadState $s['StderrTask'] 'TestReaderClosed' + Test-Assert ($r.Stdout -match 'out' -and $r.Stderr -match 'err' -and $baOut -eq $true -and $baErr -eq $true) ($baPath + ": both readers are closed once a program's output ends (stdout " + $baOut + ", stderr " + $baErr + ")") +} + +# One read fails while the other stream is still held: the failure is +# reported, not just the timeout. Task.WaitAll with a timeout throws only once +# every task has finished, so it didn't throw here. +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0' +$null = $s['Proc'].WaitForExit(10000) +$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000) +$baFailed = [System.Threading.Tasks.TaskCompletionSource[string]]::new() +$baFailed.SetException([System.IO.IOException]::new('Test: the read failed.')) +$baHeld = [System.Threading.Tasks.TaskCompletionSource[string]]::new() +$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $baHeld.Task } -StreamTimeoutMs 300 -Dispose +$baHeld.SetResult('') +Test-Assert ($r.StreamTimedOut -and $r.StreamError -match 'the read failed' -and $null -eq $r.Stdout) ("a read that failed while the other stream timed out is reported in StreamError (StreamTimedOut " + $r.StreamTimedOut + ", StreamError '" + $r.StreamError + "')") +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 675310b..8a686b5 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -164,7 +164,7 @@ public static class PkState // Bump this, and the check after the type definitions, whenever any C# // in this file changes: a session keeps the types it compiled first. - public const int TypesVersion = 8; + public const int TypesVersion = 9; // Locked: Register-/Unregister-PkCancellationSource are callable from // worker runspaces, and List is not safe for concurrent writers. @@ -247,8 +247,12 @@ public sealed class PkStartedProcess // Reads one of a program's output streams to its end, in chunks, so what has // arrived can be taken at any time (Snapshot): a program the program started -// can hold the stream open long after it exits. The task's result is the -// whole text, at the end of the stream. The reader's decoder carries a +// can hold the stream open long after it exits. The first snapshot taken +// before the end is the stream's text from then on: the read carries on to +// the end, so the program holding the stream never blocks on a full pipe, +// but drops what it reads, which nobody can get any more. The task's result +// is that snapshot, or the whole text if none was taken. The reader is +// closed at the end of the stream. The reader's decoder carries a // character split across two reads over to the next, so a snapshot never // ends in half a character. Each read runs on a thread of its own // (LongRunning), not a thread-pool thread: a read blocks until every holder @@ -261,23 +265,43 @@ public sealed class PkStreamReadTask : Task private sealed class Buffer { private readonly TextReader _reader; + // Null once the text is kept: at a snapshot or at the end of the + // stream, whichever comes first. The builder goes then, so the text + // isn't held twice. private StringBuilder _text = new StringBuilder(); - private string _final; + private string _kept; private readonly object _gate = new object(); + public volatile bool ReaderClosed; public Buffer(TextReader reader) { _reader = reader; } public string Pump() { - char[] chunk = new char[4096]; - int n; - while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0) + try { - lock (_gate) { _text.Append(chunk, 0, n); } + char[] chunk = new char[4096]; + int n; + while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0) + { + lock (_gate) { if (_text != null) { _text.Append(chunk, 0, n); } } + } + return Keep(); + } + finally + { + // Closes the pipe now rather than at the next garbage collection. + try { _reader.Dispose(); ReaderClosed = true; } + catch (Exception) { } // a close that fails must not hide the read's own result or error } - // The builder goes once the text is final, so it isn't held twice. - lock (_gate) { _final = _text.ToString(); _text = null; return _final; } } // Locked: the reader may be appending while Complete-PkProcess takes it. - public string Snapshot() { lock (_gate) { return _final != null ? _final : _text.ToString(); } } + public string Keep() + { + lock (_gate) + { + if (_text != null) { _kept = _text.ToString(); _text = null; } + return _kept; + } + } + public int HeldLength { get { lock (_gate) { return _text != null ? _text.Length : _kept.Length; } } } } private readonly Buffer _buffer; @@ -290,8 +314,14 @@ public sealed class PkStreamReadTask : Task return task; } - // The text that has arrived so far; all of it once the task has finished. - public string Snapshot() { return _buffer.Snapshot(); } + // The text that has arrived so far, kept as the stream's text from then + // on; all of it if the stream has already ended. + public string Snapshot() { return _buffer.Keep(); } + + // Test-only reads: how many characters the buffer holds, and whether the + // reader has been closed. + public int TestHeldLength { get { return _buffer.HeldLength; } } + public bool TestReaderClosed { get { return _buffer.ReaderClosed; } } } // A Windows Job Object with KILL_ON_JOB_CLOSE: every child process assigned @@ -782,7 +812,7 @@ public static class PkChildProcessJob # A session that loaded an older copy keeps its compiled types. Say so, # rather than fail later with a missing-method error. Keep this number in # sync with [PkState]::TypesVersion. -if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 8) { +if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 9) { throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one." } @@ -1753,7 +1783,9 @@ function Complete-PkProcess { # that limit was hit. The stream it gave up on comes back with what arrived # before then: cut short, not lost. Its read carries on in the background # until that program lets go of it (it's killed with PowerShell at the - # latest). A stream whose read failed (StreamError) comes back $null. + # latest), dropping what arrives after. A stream whose read failed comes + # back $null, with the reason in StreamError, even if the other stream + # timed out. # .PARAMETER Job # What Start-PkProcess returned, or a dispatcher job with the same keys. # .PARAMETER StreamTimeoutMs @@ -1793,12 +1825,20 @@ function Complete-PkProcess { } catch { $result.StreamError = [string]$_.Exception.Message } + # WaitAll throws only once every task has finished, so a read that + # failed while the other stream was still held open shows up here. + if ($null -eq $result.StreamError) { + foreach ($task in $tasks) { + if ($task.IsFaulted) { $result.StreamError = [string]$task.Exception.GetBaseException().Message; break } + } + } } # The output's end (both streams closed, or the timeout) is separate from # the program's end (the exit code, below): a program it started can hold # a stream open after it exits. A stream still open at the timeout gives # the text that arrived before it, each stream on its own; its read - # carries on in the background until that program lets go. + # carries on in the background until that program lets go, dropping + # what arrives after (Snapshot keeps the text it returns). $ranToCompletion = [System.Threading.Tasks.TaskStatus]::RanToCompletion foreach ($stream in 'Stdout', 'Stderr') { $task = $Job[$stream + 'Task'] diff --git a/README.md b/README.md index a867422..0c46e84 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. -- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it. `StreamError` means a read failed, and that stream comes back `$null`. +- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`. From 130c60b9b4d9e0517406b83fb64c1d7aa65f6668 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:42:03 -0500 Subject: [PATCH 2/2] StreamError gives the failed read's own reason; document the kept text StreamError now comes from the faulted read task in every case, rather than PowerShell's wrapper around Task.WaitAll ("One or more errors occurred"). The help and README say that a second Complete-PkProcess on the same job returns the same text, as does the read task's Result. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 9 +++++++++ ParallelKit.ps1 | 19 ++++++++++--------- README.md | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 4da1d01..aa1074c 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4568,7 +4568,9 @@ try { Start-Sleep -Seconds 3 $baHeldLater = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength' $baStillReading = -not $s['StdoutTask'].IsCompleted + $baAgain = Complete-PkProcess $s -StreamTimeoutMs 0 $baLength = ([string]$r.Stdout).Length + Test-Assert ($baAgain.Stdout -ceq $r.Stdout) ("a second Complete-PkProcess on the same job returns the same text (" + ([string]$baAgain.Stdout).Length + " characters, first " + $baLength + ")") Test-Assert ($r.StreamTimedOut -and $r.Stdout -match 'first' -and $baStillReading -and $baHeldAtResult -eq $baLength -and $baHeldLater -eq $baLength) ("a program that keeps writing after the result: the buffer stays at the result's " + $baLength + " characters (at the result " + $baHeldAtResult + ", 3 s later " + $baHeldLater + ", still reading " + $baStillReading + ")") } finally { Stop-HarnessPingsOf -ParentId $s['Proc'].Id } $baEnded = $s['StdoutTask'].Wait(10000) -and $s['StderrTask'].Wait(10000) @@ -4600,6 +4602,13 @@ $baHeld = [System.Threading.Tasks.TaskCompletionSource[string]]::new() $r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $baHeld.Task } -StreamTimeoutMs 300 -Dispose $baHeld.SetResult('') Test-Assert ($r.StreamTimedOut -and $r.StreamError -match 'the read failed' -and $null -eq $r.Stdout) ("a read that failed while the other stream timed out is reported in StreamError (StreamTimedOut " + $r.StreamTimedOut + ", StreamError '" + $r.StreamError + "')") +# And when the other stream ended normally: the read's own reason, not +# WaitAll's "One or more errors occurred". +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0' +$null = $s['Proc'].WaitForExit(10000) +$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000) +$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $s['StderrTask'] } -StreamTimeoutMs 300 -Dispose +Test-Assert (-not $r.StreamTimedOut -and $r.StreamError -eq 'Test: the read failed.' -and $null -eq $r.Stdout) ("a failed read next to one that ended gives the read's own reason (StreamError '" + $r.StreamError + "')") Write-Host "" Set-PkMessageHandler -Handler $null diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 8a686b5..fb5ca71 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1783,9 +1783,10 @@ function Complete-PkProcess { # that limit was hit. The stream it gave up on comes back with what arrived # before then: cut short, not lost. Its read carries on in the background # until that program lets go of it (it's killed with PowerShell at the - # latest), dropping what arrives after. A stream whose read failed comes - # back $null, with the reason in StreamError, even if the other stream - # timed out. + # latest), dropping what arrives after: the text handed over is that + # stream's text from then on, and calling this again on the same job + # returns the same. A stream whose read failed comes back $null, with the + # reason in StreamError, even if the other stream timed out. # .PARAMETER Job # What Start-PkProcess returned, or a dispatcher job with the same keys. # .PARAMETER StreamTimeoutMs @@ -1825,12 +1826,12 @@ function Complete-PkProcess { } catch { $result.StreamError = [string]$_.Exception.Message } - # WaitAll throws only once every task has finished, so a read that - # failed while the other stream was still held open shows up here. - if ($null -eq $result.StreamError) { - foreach ($task in $tasks) { - if ($task.IsFaulted) { $result.StreamError = [string]$task.Exception.GetBaseException().Message; break } - } + # The failed read's own reason, rather than WaitAll's wrapper. Read + # from the tasks: WaitAll throws only once every task has finished, + # so it says nothing of a read that failed while the other stream was + # still held open. + foreach ($task in $tasks) { + if ($task.IsFaulted) { $result.StreamError = [string]$task.Exception.GetBaseException().Message; break } } } # The output's end (both streams closed, or the timeout) is separate from diff --git a/README.md b/README.md index 0c46e84..0e26743 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. -- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. +- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`.