From 3e496a8368387b3be838d485171e951b0aa0e178 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:23:48 -0500 Subject: [PATCH 1/5] Add -StreamTimeoutMs to Invoke-PkProcessBatch and Invoke-PkProcessDispatch Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 68 ++++++++++++++++++++++++++++++++++++ ParallelKit.ps1 | 76 ++++++++++++++++++++++++++++------------- README.md | 6 ++-- 3 files changed, 123 insertions(+), 27 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 61ea44e..191a226 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4726,6 +4726,74 @@ try { $bcOk = (Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/d /c e Test-Assert ($bcOk -match '^ok') ("a program runs with -OutputEncoding '65001' ('" + ([string]$bcOk).Trim() + "')") Write-Host "" +Write-Host "=== BD: -StreamTimeoutMs on the batch and the dispatcher ===" -ForegroundColor Cyan +# The wait for held output was fixed at 15 s in both, and since the wait +# moved into the dispatcher's default poll, a shorter -StreamTimeoutMs passed +# to Complete-PkProcess in -JobFinalize no longer shortened it. Each held +# command leaves a ping (-w 4321, BB's helpers) holding its output for 8 s. +Reset-Harness +function Invoke-HarnessHeldDispatch { + param([int]$StreamTimeoutMs, $FinalizeTimeoutMs) + $feed = @{ Given = $false } + $got = @{} + $jobs = @(New-HarnessHeldCommand -Seconds 8 -Key 'held') + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $null = Invoke-PkProcessDispatch -MaxConcurrent 1 -StreamTimeoutMs $StreamTimeoutMs ` + -GroupProvider { if ($feed.Given) { return $null }; $feed.Given = $true; return @{ Name = 'held' } }.GetNewClosure() ` + -JobBuilder { param($g) return $jobs }.GetNewClosure() ` + -JobFinalize { + param($j, $q) + if ($null -eq $FinalizeTimeoutMs) { $got.R = Complete-PkProcess -Job $j } else { $got.R = Complete-PkProcess -Job $j -StreamTimeoutMs $FinalizeTimeoutMs } + }.GetNewClosure() + $sw.Stop() + return @{ Result = $got.R; Seconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) } +} +try { + $bdOut = $null; $bdError = $null + $sw = [System.Diagnostics.Stopwatch]::StartNew() + try { $bdOut = Invoke-PkProcessBatch -StreamTimeoutMs 1000 -Commands @(New-HarnessHeldCommand -Seconds 8 -Key 'held') } catch { $bdError = [string]$_ } + $sw.Stop() + $bdR = $null; if ($null -ne $bdOut) { $bdR = @($bdOut.Results)[0] } + Test-Assert ($null -ne $bdR -and $bdR.StreamTimedOut -and $bdR.Stdout -match 'held' -and $sw.Elapsed.TotalSeconds -lt 4) ("batch -StreamTimeoutMs 1000: held output is given up on after about a second, with what arrived (" + [math]::Round($sw.Elapsed.TotalSeconds, 1) + " s" + $(if ($bdError) { ", error: " + $bdError }) + ")") + Stop-HarnessTaggedPings + + $bdOut = $null; $bdError = $null + $sw = [System.Diagnostics.Stopwatch]::StartNew() + try { $bdOut = Invoke-PkProcessBatch -StreamTimeoutMs 0 -Commands @(New-HarnessHeldCommand -Seconds 8 -Key 'held') } catch { $bdError = [string]$_ } + $sw.Stop() + $bdR = $null; if ($null -ne $bdOut) { $bdR = @($bdOut.Results)[0] } + Test-Assert ($null -ne $bdR -and $bdR.StreamTimedOut -and $sw.Elapsed.TotalSeconds -lt 3) ("batch -StreamTimeoutMs 0: doesn't wait for held output (" + [math]::Round($sw.Elapsed.TotalSeconds, 1) + " s" + $(if ($bdError) { ", error: " + $bdError }) + ")") + Stop-HarnessTaggedPings + + # The dispatcher: Complete-PkProcess with no -StreamTimeoutMs of its own + # uses the dispatcher's, so it doesn't wait again; one with its own waits + # only the rest. + $bdRun = $null; $bdError = $null + try { $bdRun = Invoke-HarnessHeldDispatch -StreamTimeoutMs 1000 -FinalizeTimeoutMs $null } catch { $bdError = [string]$_ } + Test-Assert ($null -ne $bdRun -and $bdRun.Result.StreamTimedOut -and $bdRun.Result.Stdout -match 'held' -and $bdRun.Seconds -lt 4) ("dispatcher -StreamTimeoutMs 1000: a plain Complete-PkProcess in -JobFinalize doesn't wait a second time (" + $(if ($bdRun) { [string]$bdRun.Seconds + " s" } else { $bdError }) + ")") + Stop-HarnessTaggedPings + $bdRun = $null; $bdError = $null + try { $bdRun = Invoke-HarnessHeldDispatch -StreamTimeoutMs 1000 -FinalizeTimeoutMs 3000 } catch { $bdError = [string]$_ } + Test-Assert ($null -ne $bdRun -and $bdRun.Result.StreamTimedOut -and $bdRun.Seconds -ge 2.5 -and $bdRun.Seconds -lt 6) ("dispatcher -StreamTimeoutMs 1000, Complete-PkProcess -StreamTimeoutMs 3000: waits the rest, about 3 s in all (" + $(if ($bdRun) { [string]$bdRun.Seconds + " s" } else { $bdError }) + ")") + Stop-HarnessTaggedPings + + # A negative value is refused before anything starts. + $bdStarted = @{ Count = 0 } + $bdErrors = @() + foreach ($bdWhich in 'batch', 'dispatcher') { + $bdError = $null + try { + if ($bdWhich -eq 'batch') { $null = Invoke-PkProcessBatch -StreamTimeoutMs -1 -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } + else { $null = Invoke-PkProcessDispatch -MaxConcurrent 1 -StreamTimeoutMs -1 -GroupProvider { $bdStarted.Count++; return $null }.GetNewClosure() -JobBuilder { param($g) } -JobFinalize { param($j, $q) } } + } catch { $bdError = [string]$_ } + $bdErrors += $bdError + } + Test-Assert ($bdErrors[0] -like 'Invoke-PkProcessBatch: -StreamTimeoutMs must be 0 or more (got -1)*' -and $bdErrors[1] -like 'Invoke-PkProcessDispatch: -StreamTimeoutMs must be 0 or more (got -1)*' -and $bdStarted.Count -eq 0) ("-StreamTimeoutMs -1 is refused by both before anything starts (" + ($bdErrors -join ' | ') + ")") +} finally { + Stop-HarnessTaggedPings +} +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 003ee1d..f223f04 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1798,7 +1798,8 @@ function Complete-PkProcess { # How long to wait for the output after the program exits, # in milliseconds (default 15000); 0 or more, and 0 doesn't wait. On a job # the dispatcher's default -JobPollCompleted has already waited on, only - # what's left of it, so with the default nothing. + # what's left of it after that wait. Not given there, it's the + # dispatcher's own -StreamTimeoutMs, so nothing is left. # .PARAMETER Dispose # Dispose the process object afterwards. # .OUTPUTS @@ -1811,7 +1812,8 @@ function Complete-PkProcess { param( [Parameter(Mandatory, Position = 0)] [hashtable]$Job, - # The dispatcher's default -JobPollCompleted waits the same 15 s: keep the two in step. + # Invoke-PkProcessDispatch's and Invoke-PkProcessBatch's -StreamTimeoutMs + # default to the same 15000: keep the three in step. [int]$StreamTimeoutMs = 15000, [switch]$Dispose ) @@ -1833,8 +1835,12 @@ function Complete-PkProcess { # final pass after an abort's wait sets OutputWaitOver: at most 250 ms # then, enough for a program that exited just before to have its output # read to the end, short enough for a window close. - $waitMs = $StreamTimeoutMs - if ($Job['OutputWait'] -is [System.Diagnostics.Stopwatch]) { $waitMs = [int][Math]::Max(0, $StreamTimeoutMs - $Job['OutputWait'].ElapsedMilliseconds) } + # OutputWaitMs is the dispatcher's -StreamTimeoutMs, used when the caller + # passes none of its own. + $limitMs = $StreamTimeoutMs + if (-not $PSBoundParameters.ContainsKey('StreamTimeoutMs') -and $Job['OutputWaitMs'] -is [int]) { $limitMs = $Job['OutputWaitMs'] } + $waitMs = $limitMs + if ($Job['OutputWait'] -is [System.Diagnostics.Stopwatch]) { $waitMs = [int][Math]::Max(0, $limitMs - $Job['OutputWait'].ElapsedMilliseconds) } if ($Job['OutputWaitOver']) { $waitMs = [Math]::Min($waitMs, 250) } if ($tasks.Count -gt 0) { try { @@ -1961,7 +1967,7 @@ function Invoke-PkProcessDispatch { # The dispatcher keeps its own state on each job hashtable. Don't use these keys # for anything else: GroupKey, SlotReleased, Proc, StdoutTask, StderrTask, # Stopwatch, TimeLimit, TimedOut, StartFailureReported, OutputWait, - # OutputWaitOver. It also sets Group to + # OutputWaitOver, OutputWaitMs. It also sets Group to # the job's group, unless the job already has a Group of its own. The default # starter reads FilePath, Arguments (a string as written, or a list quoted one # argument per element), WorkingDirectory and OutputEncoding from the job. A @@ -1979,8 +1985,8 @@ function Invoke-PkProcessDispatch { # .PARAMETER JobPollCompleted # { param($job) } returning $true once the job is finished. Only its last # output value counts. Default: once the program has exited and its output - # has been read, or 15 s after the exit if a program it started holds the - # output open (the job keeps its slot meanwhile), so Complete-PkProcess in + # has been read, or -StreamTimeoutMs after the exit if a program it started + # holds the output open (the job keeps its slot meanwhile), so Complete-PkProcess in # -JobFinalize doesn't wait and the other jobs carry on. # .PARAMETER JobFinalize # { param($job, $pendingQueue) } called once per finished job; collect its @@ -2016,6 +2022,10 @@ function Invoke-PkProcessDispatch { # (default 0, no limit). A job's own TimeoutSeconds key overrides it, and 0 there # opts the job out. The killed job still goes through -JobFinalize, with # $job['TimedOut'] = $true. A job whose .Proc isn't a process isn't timed. + # .PARAMETER StreamTimeoutMs + # How long the default -JobPollCompleted waits for a job's output after its + # program exits, in milliseconds (default 15000); 0 or more, and 0 doesn't + # wait. Complete-PkProcess in -JobFinalize uses it too, unless given its own. # .OUTPUTS # An object with GroupResults, Aborted, ForceKilledCount and # JobStarterFailedCount. Each group result has GroupInfo (the group), Completed @@ -2041,7 +2051,8 @@ function Invoke-PkProcessDispatch { [int]$AbortTimeoutSeconds = 60, [int]$MaxQueueAhead = 3, [scriptblock]$GroupReleaseCheck = $null, - [double]$TimeoutSeconds = 0 + [double]$TimeoutSeconds = 0, + [int]$StreamTimeoutMs = 15000 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't # a module, so a callback without .GetNewClosure() sees this frame's @@ -2060,7 +2071,8 @@ function Invoke-PkProcessDispatch { $pkMaxQueueAhead = $MaxQueueAhead $pkGroupReleaseCheck = $GroupReleaseCheck $pkTimeoutSeconds = $TimeoutSeconds - Remove-Variable -Name GroupProvider, JobBuilder, JobStarter, JobPollCompleted, JobFinalize, JobAbortCleanup, GroupCompleteCallback, MaxConcurrent, PollCallback, CancelToken, AbortTimeoutSeconds, MaxQueueAhead, GroupReleaseCheck, TimeoutSeconds -Scope Local + $pkStreamTimeoutMs = $StreamTimeoutMs + Remove-Variable -Name GroupProvider, JobBuilder, JobStarter, JobPollCompleted, JobFinalize, JobAbortCleanup, GroupCompleteCallback, MaxConcurrent, PollCallback, CancelToken, AbortTimeoutSeconds, MaxQueueAhead, GroupReleaseCheck, TimeoutSeconds, StreamTimeoutMs -Scope Local # 0 or less would never start a job (or pull a group) and spin forever. if ($pkMaxConcurrent -isnot [System.Threading.SemaphoreSlim]) { @@ -2071,9 +2083,9 @@ function Invoke-PkProcessDispatch { $pkMaxConcurrent = $pkMaxConcurrentInt } if ($pkMaxQueueAhead -lt 1) { throw "Invoke-PkProcessDispatch: -MaxQueueAhead must be at least 1." } - # Invoke-PkProcessBatch makes the same -MaxConcurrent, -TimeoutSeconds and - # -AbortTimeoutSeconds checks, so its errors name the batch: keep the two in - # step. NaN and +Infinity pass a `-lt 0` test and acted as no limit. A job's + # Invoke-PkProcessBatch makes the same -MaxConcurrent, -TimeoutSeconds, + # -AbortTimeoutSeconds and -StreamTimeoutMs checks, so its errors name the + # batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and acted as no limit. A job's # own TimeoutSeconds key follows the same rule (Get-PkJobTimeLimit). if ([double]::IsNaN($pkTimeoutSeconds) -or [double]::IsInfinity($pkTimeoutSeconds) -or $pkTimeoutSeconds -lt 0) { throw ("Invoke-PkProcessDispatch: -TimeoutSeconds must be a number of seconds, 0 (no limit) or more (got " + $pkTimeoutSeconds + ").") @@ -2081,6 +2093,9 @@ function Invoke-PkProcessDispatch { if ($pkAbortTimeoutSeconds -lt 0) { throw ("Invoke-PkProcessDispatch: -AbortTimeoutSeconds must be 0 or more (got " + $pkAbortTimeoutSeconds + "). It's how long an abort waits for running jobs before force-stopping them; 0 doesn't wait.") } + if ($pkStreamTimeoutMs -lt 0) { + throw ("Invoke-PkProcessDispatch: -StreamTimeoutMs must be 0 or more (got " + $pkStreamTimeoutMs + "). It's how long to wait for a job's output after its program exits, in milliseconds; 0 doesn't wait.") + } # Size the thread pool up front. For a shared budget, its free slots now # are the best measure of its size. @@ -2107,20 +2122,23 @@ function Invoke-PkProcessDispatch { # Done once the program has exited and its output has been read, so # Complete-PkProcess in -JobFinalize has nothing to wait for: waiting # there would stall the whole loop. A job whose output a leftover - # program holds open keeps its slot until the reads end or 15 s pass - # (Complete-PkProcess's default -StreamTimeoutMs; keep the two in - # step), timed by OutputWait, which Complete-PkProcess reads. After an - # abort's wait the final pass sets OutputWaitOver: it takes what has - # arrived. + # program holds open keeps its slot until the reads end or + # -StreamTimeoutMs passes, timed by OutputWait. The limit goes on the + # job as OutputWaitMs, so a Complete-PkProcess given no -StreamTimeoutMs + # of its own waits only what's left of it (nothing). After an abort's + # wait the final pass sets OutputWaitOver: it takes what has arrived. + # A closure, for $pkStreamTimeoutMs: a callback here only sees this + # frame's variables through dynamic scope, which a user's own + # variables could shadow. $pkJobPollCompleted = { param($pkJob) if (-not $pkJob['Proc'].HasExited) { return $false } $pkOut = $pkJob['StdoutTask']; $pkErr = $pkJob['StderrTask'] if (($pkOut -isnot [System.Threading.Tasks.Task] -or $pkOut.IsCompleted) -and ($pkErr -isnot [System.Threading.Tasks.Task] -or $pkErr.IsCompleted)) { return $true } if ($pkJob['OutputWaitOver']) { return $true } - if ($pkJob['OutputWait'] -isnot [System.Diagnostics.Stopwatch]) { $pkJob['OutputWait'] = [System.Diagnostics.Stopwatch]::StartNew() } - return ($pkJob['OutputWait'].ElapsedMilliseconds -ge 15000) - } + if ($pkJob['OutputWait'] -isnot [System.Diagnostics.Stopwatch]) { $pkJob['OutputWaitMs'] = $pkStreamTimeoutMs; $pkJob['OutputWait'] = [System.Diagnostics.Stopwatch]::StartNew() } + return ($pkJob['OutputWait'].ElapsedMilliseconds -ge $pkJob['OutputWaitMs']) + }.GetNewClosure() } if ($null -eq $pkJobAbortCleanup) { $pkJobAbortCleanup = { param($pkJob) } } if ($null -eq $pkGroupCompleteCallback) { $pkGroupCompleteCallback = { param($pkGroup) } } @@ -2339,7 +2357,7 @@ function Invoke-PkProcessDispatchCore { # its old, disposed process: cleared first. if ($pkJob['SlotReleased']) { $pkJob['Proc'] = $null; $pkJob['StdoutTask'] = $null; $pkJob['StderrTask'] = $null; $pkJob['Stopwatch'] = $null - $pkJob['OutputWait'] = $null; $pkJob['OutputWaitOver'] = $null + $pkJob['OutputWait'] = $null; $pkJob['OutputWaitOver'] = $null; $pkJob['OutputWaitMs'] = $null } $pkJob['SlotReleased'] = $false $pkStarting = $pkJob @@ -2714,6 +2732,10 @@ function Invoke-PkProcessBatch { # .PARAMETER OutputEncoding # How to decode every program's output: a name like 'utf-8', a code page number # (65001 or '65001'), or a [System.Text.Encoding]. Default: the console's code page. + # .PARAMETER StreamTimeoutMs + # How long to wait for a program's output after it exits, in milliseconds + # (default 15000); 0 or more, and 0 doesn't wait. Past it, the result has + # StreamTimedOut = $true and the output that arrived. # .OUTPUTS # An object with Results, Aborted and ForceKilledCount. Each result has Command, # ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, StartError, Aborted, @@ -2733,7 +2755,8 @@ function Invoke-PkProcessBatch { [Alias('DrainTimeoutSeconds')] [int]$AbortTimeoutSeconds = 60, [double]$TimeoutSeconds = 0, - $OutputEncoding = $null + $OutputEncoding = $null, + [int]$StreamTimeoutMs = 15000 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't # a module, so a callback without .GetNewClosure() sees this frame's @@ -2744,11 +2767,12 @@ function Invoke-PkProcessBatch { $pkAbortTimeoutSeconds = $AbortTimeoutSeconds $pkTimeoutSeconds = $TimeoutSeconds $pkOutputEncoding = $OutputEncoding - Remove-Variable -Name Commands, MaxConcurrent, OnComplete, AbortTimeoutSeconds, TimeoutSeconds, OutputEncoding -Scope Local + $pkStreamTimeoutMs = $StreamTimeoutMs + Remove-Variable -Name Commands, MaxConcurrent, OnComplete, AbortTimeoutSeconds, TimeoutSeconds, OutputEncoding, StreamTimeoutMs -Scope Local # Checked once here, so a bad name fails the call instead of every command. $null = Resolve-PkEncoding -Encoding $pkOutputEncoding - # Invoke-PkProcessDispatch makes the same three checks; these name the + # Invoke-PkProcessDispatch makes the same four checks; these name the # batch. Keep the two in step. Get-PkJobTimeLimit checks a job's own # TimeoutSeconds key by the same rule. if ($pkMaxConcurrent -lt 1) { throw ("Invoke-PkProcessBatch: -MaxConcurrent must be at least 1 (got " + $pkMaxConcurrent + ").") } @@ -2758,6 +2782,9 @@ function Invoke-PkProcessBatch { if ($pkAbortTimeoutSeconds -lt 0) { throw ("Invoke-PkProcessBatch: -AbortTimeoutSeconds must be 0 or more (got " + $pkAbortTimeoutSeconds + "). It's how long an abort waits for running programs before force-stopping them; 0 doesn't wait.") } + if ($pkStreamTimeoutMs -lt 0) { + throw ("Invoke-PkProcessBatch: -StreamTimeoutMs must be 0 or more (got " + $pkStreamTimeoutMs + "). It's how long to wait for a program's output after it exits, in milliseconds; 0 doesn't wait.") + } $pkResults = [System.Collections.Generic.List[object]]::new() $pkState = @{ Given = $false } @@ -2856,6 +2883,7 @@ function Invoke-PkProcessBatch { GroupProvider = $pkGroupProvider; JobBuilder = $pkJobBuilder; JobStarter = $pkJobStarter JobFinalize = $pkJobFinalize; JobAbortCleanup = $pkJobAbortCleanup; MaxConcurrent = $pkMaxConcurrent AbortTimeoutSeconds = $pkAbortTimeoutSeconds; MaxQueueAhead = 1; TimeoutSeconds = $pkTimeoutSeconds + StreamTimeoutMs = $pkStreamTimeoutMs } try { $pkDispatch = Invoke-PkProcessDispatch @pkDispatchParams diff --git a/README.md b/README.md index bd9aab8..489e8e0 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. -- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. While a program's output is held open, its command keeps its slot, but the other commands carry on: they keep starting, finishing and being killed at their time limits, and an abort is acted on at once. If an abort's wait for running programs ends while one is still waiting for its output, it gets what has arrived, with `StreamTimedOut`, rather than counting as force-stopped. +- `StreamTimedOut` means the output read gave up 15 seconds after the program exited (`-StreamTimeoutMs` changes that, in milliseconds; 0 doesn't wait), on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. While a program's output is held open, its command keeps its slot, but the other commands carry on: they keep starting, finishing and being killed at their time limits, and an abort is acted on at once. If an abort's wait for running programs ends while one is still waiting for its output, it gets what has arrived, with `StreamTimedOut`, rather than counting as force-stopped. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`. @@ -249,11 +249,11 @@ $dispatch = Invoke-PkProcessDispatch @dispatchParams All of these callbacks run on whichever thread called the dispatcher: the main thread normally, or the worker if you call it from inside a runspace job (in which case the worker rules apply to them). - **`-GroupCompleteCallback` also runs for a group an abort cut short**, so you can record the jobs in it that did finish. Before doing anything irreversible there (zipping the folder, deleting sources), check `Test-PkCancelRequested` (or your own `-CancelToken`) and skip it if true. -- `-JobFinalize` is called once per finished job. Collect its exit code and output there with `Complete-PkProcess -Job $job`; the dispatcher disposes the process right after. With the default `-JobPollCompleted`, a job is finished once its program has exited and its output has been read, or 15 seconds after the exit when a program it started holds the output open (the job keeps its slot meanwhile, and the other jobs carry on), so `Complete-PkProcess` there has nothing left to wait for. A longer `-StreamTimeoutMs` there waits only the rest, but on the dispatch loop's own thread, holding up the other jobs; a shorter one can't shorten the poll's 15 seconds. With your own `-JobPollCompleted`, `Complete-PkProcess` waits as it does anywhere else. +- `-JobFinalize` is called once per finished job. Collect its exit code and output there with `Complete-PkProcess -Job $job`; the dispatcher disposes the process right after. With the default `-JobPollCompleted`, a job is finished once its program has exited and its output has been read, or `-StreamTimeoutMs` after the exit (15 seconds by default) when a program it started holds the output open (the job keeps its slot meanwhile, and the other jobs carry on), so `Complete-PkProcess` there has nothing left to wait for: given no `-StreamTimeoutMs` of its own, it uses the dispatcher's. A longer one there waits only the rest, but on the dispatch loop's own thread, holding up the other jobs; to wait less, set the dispatcher's `-StreamTimeoutMs` instead. With your own `-JobPollCompleted`, `Complete-PkProcess` waits as it does anywhere else. - Groups are pulled lazily, at most `-MaxQueueAhead` (default 3) ahead. - A group can be any value: a hashtable, a path, a name. Each job gets its group as `$job.Group`. - Each `-GroupProvider` call outputs exactly one group, or `$null` to end, and nothing else: `return $group`, `return ,@('a', 'b', 'c')` for a list as one group (the comma keeps a one-item list a list), or `return $null`. Anything else it lets out (an `ArrayList.Add` index, say) becomes part of the group, so discard it with `$null = ...`. Output that ends in `$null` is taken as the end, with a warning, so a list group can't end in `$null`. -- The dispatcher keeps its state on the job hashtables. Don't use these names for your own job fields: `GroupKey`, `SlotReleased`, `Proc`, `StdoutTask`, `StderrTask`, `Stopwatch`, `TimeLimit`, `TimedOut`, `StartFailureReported`, `OutputWait`, `OutputWaitOver`. (`Group` is yours if you set it yourself.) +- The dispatcher keeps its state on the job hashtables. Don't use these names for your own job fields: `GroupKey`, `SlotReleased`, `Proc`, `StdoutTask`, `StderrTask`, `Stopwatch`, `TimeLimit`, `TimedOut`, `StartFailureReported`, `OutputWait`, `OutputWaitOver`, `OutputWaitMs`. (`Group` is yours if you set it yourself.) - `-TimeoutSeconds 600` kills a job's program, and anything it started, if it's still running after 10 minutes. The default, 0, means no limit, and it can be fractional. A job's own `TimeoutSeconds` key overrides it; 0 opts the job out, and a bad value is a failed start. The killed job still goes through `-JobFinalize`, where `$job['TimedOut']` is `$true`, and other jobs carry on. The limit stays in force while an abort waits for running jobs. A retry gets a fresh clock. A job whose `.Proc` isn't a process isn't timed: `-JobPollCompleted` is the place for its limit. - To retry a job or add follow-up work, enqueue job hashtables onto `$pending` inside `-JobFinalize`. They join the same group, and the group isn't complete until they've all finished. A retry can start within milliseconds of the failure, so cap the number of retries, and don't count on the gap for something like a locked file to clear. - Supply `-JobStarter { param($job) ... }` to start jobs yourself. It must set `$job.Proc` and should return the job. A starter that is just `Start-PkProcess ...` works too: its result is folded into your job. Throw to report a job that couldn't start. From 575e6b1d19aa44a803db10f4d55308312523a1e6 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:41:32 -0500 Subject: [PATCH 2/5] Ignore a stray OutputWaitMs, test a re-queued job and a worker, note the shared default Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 32 ++++++++++++++++++++++++++++++++ ParallelKit.ps1 | 28 ++++++++++++++++++---------- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 68f09b1..eaed3ef 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4790,6 +4790,38 @@ try { Test-Assert ($null -ne $bdRun -and $bdRun.Result.StreamTimedOut -and $bdRun.Seconds -ge 2.5 -and $bdRun.Seconds -lt 6) ("dispatcher -StreamTimeoutMs 1000, Complete-PkProcess -StreamTimeoutMs 3000: waits the rest, about 3 s in all (" + $(if ($bdRun) { [string]$bdRun.Seconds + " s" } else { $bdError }) + ")") Stop-HarnessTaggedPings + # A job -JobFinalize re-queues waits the dispatcher's limit again, from + # its own new exit. + $bdFeed = @{ Given = $false } + $bdGot = [System.Collections.Generic.List[object]]::new() + $bdJobs = @(New-HarnessHeldCommand -Seconds 8 -Key 'again') + $bdError = $null + $sw = [System.Diagnostics.Stopwatch]::StartNew() + try { + $null = Invoke-PkProcessDispatch -MaxConcurrent 1 -StreamTimeoutMs 1000 ` + -GroupProvider { if ($bdFeed.Given) { return $null }; $bdFeed.Given = $true; return @{ Name = 'again' } }.GetNewClosure() ` + -JobBuilder { param($g) return $bdJobs }.GetNewClosure() ` + -JobFinalize { param($j, $q) $bdGot.Add((Complete-PkProcess -Job $j)); if ($bdGot.Count -eq 1) { $q.Enqueue($j) } }.GetNewClosure() + } catch { $bdError = [string]$_ } + $sw.Stop() + $bdBoth = @($bdGot | Where-Object { $_.StreamTimedOut -and $_.Stdout -match 'held' }).Count + Test-Assert ($bdGot.Count -eq 2 -and $bdBoth -eq 2 -and $sw.Elapsed.TotalSeconds -lt 5) ("dispatcher -StreamTimeoutMs 1000: a re-queued job is given up on after about a second again (" + $bdBoth + " of " + $bdGot.Count + " runs timed out with their output, " + [math]::Round($sw.Elapsed.TotalSeconds, 1) + " s" + $(if ($bdError) { ", error: " + $bdError }) + ")") + Stop-HarnessTaggedPings + + # Inside a worker the kit isn't a module: the default poll still carries + # the limit it was given. + $bdRes = [System.Collections.Generic.List[object]]::new() + $bdOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(@{ Cmd = $cmdExe }) -ResultHandler { param($r) $bdRes.Add($r) }.GetNewClosure() -WorkerScript { + param($Cmd) + $wsw = [System.Diagnostics.Stopwatch]::StartNew() + $wb = Invoke-PkProcessBatch -StreamTimeoutMs 1000 -Commands @(@{ FilePath = $Cmd; Arguments = '/d /c echo held& start /b ping -n 9 -w 4321 127.0.0.1 >nul' }) + $wr = @($wb.Results)[0] + [pscustomobject]@{ TimedOut = $wr.StreamTimedOut; Held = ($wr.Stdout -match 'held'); Seconds = [math]::Round($wsw.Elapsed.TotalSeconds, 1) } + } + $bdW = $null; if ($bdRes.Count -eq 1) { $bdW = $bdRes[0] } + Test-Assert ($bdOut.Errors.Count -eq 0 -and $null -ne $bdW -and $bdW.TimedOut -and $bdW.Held -and $bdW.Seconds -lt 4) ("a batch inside a worker honors -StreamTimeoutMs 1000 (" + $(if ($bdW) { [string]$bdW.Seconds + " s" } else { [string]$bdOut.Errors.Count + " errors: " + (@($bdOut.Errors | ForEach-Object { [string]$_.Error }) -join '; ') }) + ")") + Stop-HarnessTaggedPings + # A negative value is refused before anything starts. $bdStarted = @{ Count = 0 } $bdErrors = @() diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index f223f04..7464803 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1835,12 +1835,15 @@ function Complete-PkProcess { # final pass after an abort's wait sets OutputWaitOver: at most 250 ms # then, enough for a program that exited just before to have its output # read to the end, short enough for a window close. - # OutputWaitMs is the dispatcher's -StreamTimeoutMs, used when the caller - # passes none of its own. + # OutputWaitMs is the dispatcher's -StreamTimeoutMs, set with OutputWait, + # and used when the caller passes none of its own. $limitMs = $StreamTimeoutMs - if (-not $PSBoundParameters.ContainsKey('StreamTimeoutMs') -and $Job['OutputWaitMs'] -is [int]) { $limitMs = $Job['OutputWaitMs'] } - $waitMs = $limitMs - if ($Job['OutputWait'] -is [System.Diagnostics.Stopwatch]) { $waitMs = [int][Math]::Max(0, $limitMs - $Job['OutputWait'].ElapsedMilliseconds) } + if ($Job['OutputWait'] -is [System.Diagnostics.Stopwatch]) { + if (-not $PSBoundParameters.ContainsKey('StreamTimeoutMs') -and $Job['OutputWaitMs'] -is [int]) { $limitMs = $Job['OutputWaitMs'] } + $waitMs = [int][Math]::Max(0, $limitMs - $Job['OutputWait'].ElapsedMilliseconds) + } else { + $waitMs = $limitMs + } if ($Job['OutputWaitOver']) { $waitMs = [Math]::Min($waitMs, 250) } if ($tasks.Count -gt 0) { try { @@ -2052,6 +2055,8 @@ function Invoke-PkProcessDispatch { [int]$MaxQueueAhead = 3, [scriptblock]$GroupReleaseCheck = $null, [double]$TimeoutSeconds = 0, + # Complete-PkProcess's and Invoke-PkProcessBatch's -StreamTimeoutMs + # default to the same 15000: keep the three in step. [int]$StreamTimeoutMs = 15000 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't @@ -2085,8 +2090,9 @@ function Invoke-PkProcessDispatch { if ($pkMaxQueueAhead -lt 1) { throw "Invoke-PkProcessDispatch: -MaxQueueAhead must be at least 1." } # Invoke-PkProcessBatch makes the same -MaxConcurrent, -TimeoutSeconds, # -AbortTimeoutSeconds and -StreamTimeoutMs checks, so its errors name the - # batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and acted as no limit. A job's - # own TimeoutSeconds key follows the same rule (Get-PkJobTimeLimit). + # batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and + # acted as no limit. A job's own TimeoutSeconds key follows the same rule + # (Get-PkJobTimeLimit). if ([double]::IsNaN($pkTimeoutSeconds) -or [double]::IsInfinity($pkTimeoutSeconds) -or $pkTimeoutSeconds -lt 0) { throw ("Invoke-PkProcessDispatch: -TimeoutSeconds must be a number of seconds, 0 (no limit) or more (got " + $pkTimeoutSeconds + ").") } @@ -2127,9 +2133,9 @@ function Invoke-PkProcessDispatch { # job as OutputWaitMs, so a Complete-PkProcess given no -StreamTimeoutMs # of its own waits only what's left of it (nothing). After an abort's # wait the final pass sets OutputWaitOver: it takes what has arrived. - # A closure, for $pkStreamTimeoutMs: a callback here only sees this - # frame's variables through dynamic scope, which a user's own - # variables could shadow. + # A closure, so it carries this call's $pkStreamTimeoutMs itself + # rather than finding it by dynamic scope (measured: no slower per + # job than the plain block on 5.1 or 7). $pkJobPollCompleted = { param($pkJob) if (-not $pkJob['Proc'].HasExited) { return $false } @@ -2756,6 +2762,8 @@ function Invoke-PkProcessBatch { [int]$AbortTimeoutSeconds = 60, [double]$TimeoutSeconds = 0, $OutputEncoding = $null, + # Complete-PkProcess's and Invoke-PkProcessDispatch's -StreamTimeoutMs + # default to the same 15000: keep the three in step. [int]$StreamTimeoutMs = 15000 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't From a3683f3f72d48d522e63790ec50548aebac05d65 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:47:41 -0500 Subject: [PATCH 3/5] WIP: -MaxOutputChars Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 109 ++++++++++++++++++++++- ParallelKit.ps1 | 190 ++++++++++++++++++++++++++++++++-------- 2 files changed, 259 insertions(+), 40 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index eaed3ef..ef422b2 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -1353,9 +1353,9 @@ Reset-Harness function Get-HarnessShape($o) { if ($o -isnot [System.Management.Automation.PSCustomObject]) { return '(not an object)' }; return ($o.PSObject.Properties.Name -join ',') } $shapeBatch = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) Test-Assert ((Get-HarnessShape $shapeBatch) -eq 'Results,Aborted,ForceKilledCount') ("a batch returns an object (" + (Get-HarnessShape $shapeBatch) + ")") -Test-Assert ((Get-HarnessShape $shapeBatch.Results[0]) -eq 'Command,ExitCode,Stdout,Stderr,StreamTimedOut,StreamError,StartError,Aborted,TimedOut,Duration') ("each batch result is an object, fields in order (" + (Get-HarnessShape $shapeBatch.Results[0]) + ")") +Test-Assert ((Get-HarnessShape $shapeBatch.Results[0]) -eq 'Command,ExitCode,Stdout,Stderr,StreamTimedOut,StreamError,StartError,Aborted,TimedOut,Duration,StdoutDroppedChars,StderrDroppedChars') ("each batch result is an object, fields in order (" + (Get-HarnessShape $shapeBatch.Results[0]) + ")") $shapeOne = Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/c exit 0' -Test-Assert ((Get-HarnessShape $shapeOne) -eq 'ExitCode,Stdout,Stderr,StreamTimedOut,StreamError,Duration') ("Invoke-PkProcessWithTimeout returns an object (" + (Get-HarnessShape $shapeOne) + ")") +Test-Assert ((Get-HarnessShape $shapeOne) -eq 'ExitCode,Stdout,Stderr,StreamTimedOut,StreamError,Duration,StdoutDroppedChars,StderrDroppedChars') ("Invoke-PkProcessWithTimeout returns an object (" + (Get-HarnessShape $shapeOne) + ")") $shapeDispatch = Invoke-PkProcessDispatch -MaxConcurrent 1 -GroupProvider { return $null } -JobBuilder { param($g) } -JobFinalize { param($j, $p) } Test-Assert ((Get-HarnessShape $shapeDispatch) -eq 'GroupResults,Aborted,ForceKilledCount,JobStarterFailedCount') ("the dispatcher returns an object (" + (Get-HarnessShape $shapeDispatch) + ")") $shapeJobs = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @{ Id = 1 } -WorkerScript { param($Id) } @@ -3702,7 +3702,7 @@ function Get-HarnessUnguardedCallbacks { # this list. Invoke-PkRun's `& $ScriptBlock` is the run itself, not called # from a loop. Returns @{ Calls; Problems }. param([string]$Source) - $internal = @('$pkNewBatchResult', '$pkGetStartFailureCause', '$pkGetJobTimeLimit', '$pkReportStartFailure', '$pkStartProcessCore', '$pkWriteCallbackLoopWarning') + $internal = @('$pkNewBatchResult', '$pkGetStartFailureCause', '$pkGetJobTimeLimit', '$pkGetJobMaxOutputChars', '$pkReportStartFailure', '$pkStartProcessCore', '$pkWriteCallbackLoopWarning') $problems = [System.Collections.Generic.List[string]]::new() $tokens = $null; $errors = $null $ast = [System.Management.Automation.Language.Parser]::ParseInput($Source, [ref]$tokens, [ref]$errors) @@ -4839,6 +4839,109 @@ try { } Write-Host "" +Write-Host "=== BE: -MaxOutputChars keeps the start and the end of long output ===" -ForegroundColor Cyan +# Past the cap, each stream keeps its first half and its last half, and +# StdoutDroppedChars / StderrDroppedChars count the rest. 'echo' of 20 +# characters gives 22 with its CRLF. +Reset-Harness +function Get-HarnessCapRun { + param([string]$Arguments, [int]$Cap, $Encoding = $null) + try { return Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments $Arguments -MaxOutputChars $Cap -OutputEncoding $Encoding } catch { return [pscustomobject]@{ Stdout = $null; Stderr = $null; StdoutDroppedChars = $null; StderrDroppedChars = $null; Error = [string]$_ } } +} +function Format-HarnessCapRun($r) { + $e = ''; if ($r.PSObject.Properties['Error']) { $e = ', error: ' + $r.Error } + return ("'" + ([string]$r.Stdout -replace "`r", '\r' -replace "`n", '\n') + "', dropped " + $r.StdoutDroppedChars + $e) +} +$beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap 10 +Test-Assert ($beR.Stdout -ceq "01234HIJ`r`n" -and $beR.StdoutDroppedChars -eq 12) ("past -MaxOutputChars 10, the first 5 and the last 5 characters are kept, and 12 counted as dropped (" + (Format-HarnessCapRun $beR) + ")") +$beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap 22 +Test-Assert ($beR.Stdout -ceq "0123456789ABCDEFGHIJ`r`n" -and $beR.StdoutDroppedChars -eq 0) ("output exactly at the cap comes back whole, nothing dropped (" + (Format-HarnessCapRun $beR) + ")") +$beR = Get-HarnessCapRun -Arguments '/d /c 1>&2 echo 0123456789ABCDEFGHIJ' -Cap 10 +Test-Assert ($beR.Stderr -ceq "01234HIJ`r`n" -and $beR.StderrDroppedChars -eq 12 -and $beR.StdoutDroppedChars -eq 0) ("stderr is capped the same way, on its own ('" + ([string]$beR.Stderr -replace "`r", '\r' -replace "`n", '\n') + "', dropped " + $beR.StderrDroppedChars + ")") + +# A cut never splits a surrogate pair. cmd /u writes UTF-16: four emoji are +# 8 characters, plus CRLF. A cap of 6 puts the head's end and the tail's +# start inside pairs. +$beEmoji = [char]::ConvertFromUtf32(0x1F600) +$beR = Get-HarnessCapRun -Arguments ('/d /u /c echo ' + $beEmoji * 4) -Cap 6 -Encoding 'unicode' +$beText = [string]$beR.Stdout +$beLone = 0 +for ($i = 0; $i -lt $beText.Length; $i++) { + if ([char]::IsHighSurrogate($beText[$i])) { if ($i + 1 -lt $beText.Length -and [char]::IsLowSurrogate($beText[$i + 1])) { $i++ } else { $beLone++ } } + elseif ([char]::IsLowSurrogate($beText[$i])) { $beLone++ } +} +Test-Assert ($beText -ceq ($beEmoji + "`r`n") -and $beLone -eq 0 -and $beR.StdoutDroppedChars -eq 6) ("a cut inside a surrogate pair drops the whole pair, never half (" + $beText.Length + " characters kept, " + $beLone + " half pairs, dropped " + $beR.StdoutDroppedChars + ")") + +# Long output: what's held stays within the cap while the program writes, +# and the count adds up to the whole. +$beLoop = '/d /c for /l %i in (1,1,20000) do @echo line%i' +$beWhole = $null; try { $beWhole = Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments $beLoop -TimeoutMs 120000 } catch { } +$beHeldMax = 0; $beErr = $null; $beR = $null +try { + $beJob = Start-PkProcess -FilePath $cmdExe -Arguments $beLoop -MaxOutputChars 1000 + while (-not $beJob['Proc'].HasExited) { $beHeldMax = [Math]::Max($beHeldMax, $beJob['StdoutTask'].TestHeldLength); Start-Sleep -Milliseconds 5 } + $beR = Complete-PkProcess -Job $beJob -Dispose +} catch { $beErr = [string]$_ } +$beWholeLen = -1; if ($null -ne $beWhole) { $beWholeLen = ([string]$beWhole.Stdout).Length } +$beOk = ($null -ne $beR -and ([string]$beR.Stdout).Length -eq 1000 -and $beR.Stdout.StartsWith("line1`r`nline2`r`n") -and $beR.Stdout.EndsWith("line20000`r`n") -and ($beR.StdoutDroppedChars + 1000) -eq $beWholeLen) +Test-Assert ($beOk -and $beHeldMax -le 1000) ("20,000 lines under a 1,000-character cap: 1,000 kept, from line1 to line20000, the dropped count adds up to the uncapped " + $beWholeLen + ", and at most " + $beHeldMax + " held while it ran" + $(if ($beErr) { " (error: " + $beErr + ")" })) + +# A stream timeout takes the start and the end of what had arrived. +try { + $beJob = $null; $beR = $null; $beErr = $null + try { + $beJob = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo 0123456789ABCDEFGHIJ& start /b ping -n 9 -w 4321 127.0.0.1 >nul' -MaxOutputChars 10 + $null = Wait-PkProcess -Process $beJob['Proc'] -TimeoutMs 10000 + $beR = Complete-PkProcess -Job $beJob -StreamTimeoutMs 500 -Dispose + } catch { $beErr = [string]$_ } + Test-Assert ($null -ne $beR -and $beR.StreamTimedOut -and $beR.Stdout -ceq "01234HIJ`r`n" -and $beR.StdoutDroppedChars -eq 12) ("at a stream timeout, the capped text so far is handed over, with its count (" + $(if ($beR) { (Format-HarnessCapRun $beR) + ", StreamTimedOut " + $beR.StreamTimedOut } else { $beErr }) + ")") +} finally { Stop-HarnessTaggedPings } + +# The batch: -MaxOutputChars for every command, a command's own key over it +# (0 opts out), a bad key is that command's start error. +$beB = $null; $beErr = $null +try { + $beB = Invoke-PkProcessBatch -MaxOutputChars 10 -Commands @( + @{ FilePath = $cmdExe; Arguments = '/d /c echo 0123456789ABCDEFGHIJ'; Key = 'capped' }, + @{ FilePath = $cmdExe; Arguments = '/d /c echo 0123456789ABCDEFGHIJ'; Key = 'own'; MaxOutputChars = 4 }, + @{ FilePath = $cmdExe; Arguments = '/d /c echo 0123456789ABCDEFGHIJ'; Key = 'out'; MaxOutputChars = 0 }, + @{ FilePath = $cmdExe; Arguments = '/d /c echo x'; Key = 'bad'; MaxOutputChars = 'lots' }) +} catch { $beErr = [string]$_ } +$beBy = @{}; if ($null -ne $beB) { foreach ($r in $beB.Results) { $beBy[$r.Command['Key']] = $r } } +$beText = ($beBy.Keys | Sort-Object | ForEach-Object { $_ + '=' + ([string]$beBy[$_].Stdout).Length + '/' + $beBy[$_].StdoutDroppedChars }) -join ' ' +Test-Assert ($beBy.Count -eq 4 -and $beBy['capped'].Stdout -ceq "01234HIJ`r`n" -and ([string]$beBy['own'].Stdout).Length -eq 4 -and $beBy['own'].StdoutDroppedChars -eq 18 -and ([string]$beBy['out'].Stdout).Length -eq 22 -and $beBy['out'].StdoutDroppedChars -eq 0) ("batch: -MaxOutputChars caps every command, and a command's own MaxOutputChars key overrides it, 0 opting out (" + $beText + $(if ($beErr) { ", error: " + $beErr }) + ")") +Test-Assert ($beBy.ContainsKey('bad') -and $beBy['bad'].StartError -like "MaxOutputChars must be a whole number of characters*") ("batch: a bad MaxOutputChars key is that command's start error (" + $(if ($beBy.ContainsKey('bad')) { $beBy['bad'].StartError }) + ")") + +# The dispatcher's default starter: the same, from -MaxOutputChars and the key. +$beFeed = @{ Given = $false } +$beGot = @{} +$beErr = $null +try { + $null = Invoke-PkProcessDispatch -MaxConcurrent 2 -MaxOutputChars 10 ` + -GroupProvider { if ($beFeed.Given) { return $null }; $beFeed.Given = $true; return @{ Name = 'cap' } }.GetNewClosure() ` + -JobBuilder { param($g) return @(@{ FilePath = $cmdExe; Arguments = '/d /c echo 0123456789ABCDEFGHIJ'; Key = 'capped' }, @{ FilePath = $cmdExe; Arguments = '/d /c echo 0123456789ABCDEFGHIJ'; Key = 'own'; MaxOutputChars = '4' }) }.GetNewClosure() ` + -JobFinalize { param($j, $q) $beGot[$j['Key']] = Complete-PkProcess -Job $j }.GetNewClosure() +} catch { $beErr = [string]$_ } +Test-Assert ($beGot.Count -eq 2 -and $beGot['capped'].Stdout -ceq "01234HIJ`r`n" -and ([string]$beGot['own'].Stdout).Length -eq 4 -and $beGot['own'].StdoutDroppedChars -eq 18) ("dispatcher: the default starter takes -MaxOutputChars, and a job's own key (a string here) overrides it (" + (($beGot.Keys | Sort-Object | ForEach-Object { $_ + '=' + ([string]$beGot[$_].Stdout).Length }) -join ' ') + $(if ($beErr) { ", error: " + $beErr }) + ")") + +# A negative cap is refused before anything starts. +$beErrors = @() +$beStarted = @{ Count = 0 } +foreach ($beWhich in 'Start-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessBatch', 'Invoke-PkProcessDispatch') { + $beErr = $null + try { + switch ($beWhich) { + 'Start-PkProcess' { $null = Start-PkProcess -FilePath $cmdExe -Arguments '/c exit 0' -MaxOutputChars -1 } + 'Invoke-PkProcessWithTimeout' { $null = Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/c exit 0' -MaxOutputChars -1 } + 'Invoke-PkProcessBatch' { $null = Invoke-PkProcessBatch -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) -MaxOutputChars -1 } + 'Invoke-PkProcessDispatch' { $null = Invoke-PkProcessDispatch -MaxConcurrent 1 -MaxOutputChars -1 -GroupProvider { $beStarted.Count++; return $null }.GetNewClosure() -JobBuilder { param($g) } -JobFinalize { param($j, $q) } } + } + } catch { $beErr = [string]$_ } + if ($beErr -like ($beWhich + ': -MaxOutputChars must be 0 (no cap) or more (got -1)*')) { $beErrors += 'ok' } else { $beErrors += ($beWhich + ': ' + $beErr) } +} +Test-Assert (@($beErrors | Where-Object { $_ -ne 'ok' }).Count -eq 0 -and $beStarted.Count -eq 0) ("-MaxOutputChars -1 is refused by all four before anything starts (" + ($beErrors -join ' | ') + ")") +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 7464803..1b67618 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -164,7 +164,7 @@ public static class PkState // Bump this, and the check after the type definitions, whenever any C# // in this file changes: a session keeps the types it compiled first. - public const int TypesVersion = 9; + public const int TypesVersion = 10; // Locked: Register-/Unregister-PkCancellationSource are callable from // worker runspaces, and List is not safe for concurrent writers. @@ -259,7 +259,10 @@ public sealed class PkStartedProcess // of the stream's write end has let go (at the latest when PowerShell exits: // they're in the Job Object), and leftover programs holding their output // would otherwise use up the pool, so a new program's reads would wait for -// the pool to grow and hit their stream timeout. +// the pool to grow and hit their stream timeout. With a cap (maxChars), only +// the first half and the last half of a longer stream are kept, never split +// inside a surrogate pair; DroppedChars counts the rest. The read still goes +// to the end, so the program never blocks on a full pipe. public sealed class PkStreamReadTask : Task { private sealed class Buffer @@ -267,12 +270,22 @@ public sealed class PkStreamReadTask : Task private readonly TextReader _reader; // Null once the text is kept: at a snapshot or at the end of the // stream, whichever comes first. The builder goes then, so the text - // isn't held twice. + // isn't held twice. With a cap, it holds the head (the first half). private StringBuilder _text = new StringBuilder(); private string _kept; private readonly object _gate = new object(); public volatile bool ReaderClosed; - public Buffer(TextReader reader) { _reader = reader; } + // The cap's tail: a ring of the last chars read. Null with no cap. + private readonly int _headCap; + private bool _headFull; + private readonly char[] _tail; + private int _tailStart, _tailCount; + private long _total, _dropped; + public Buffer(TextReader reader, int maxChars) + { + _reader = reader; + if (maxChars > 0) { _headCap = maxChars / 2; _tail = new char[maxChars - _headCap]; } + } public string Pump() { try @@ -281,7 +294,7 @@ public sealed class PkStreamReadTask : Task int n; while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0) { - lock (_gate) { if (_text != null) { _text.Append(chunk, 0, n); } } + lock (_gate) { if (_text != null) { Append(chunk, n); } } } return Keep(); } @@ -292,24 +305,65 @@ public sealed class PkStreamReadTask : Task catch (Exception) { } // a close that fails must not hide the read's own result or error } } + // Under _gate. + private void Append(char[] chunk, int n) + { + _total += n; + if (_tail == null) { _text.Append(chunk, 0, n); return; } + int i = 0; + if (!_headFull) + { + int take = Math.Min(_headCap - _text.Length, n); + if (take == _headCap - _text.Length) + { + _headFull = true; + // The head never ends on the first half of a surrogate pair. + if (take > 0 && Char.IsHighSurrogate(chunk[take - 1])) { take--; } + } + _text.Append(chunk, 0, take); + i = take; + } + for (; i < n; i++) + { + if (_tailCount < _tail.Length) { _tail[(_tailStart + _tailCount) % _tail.Length] = chunk[i]; _tailCount++; } + else { _tail[_tailStart] = chunk[i]; _tailStart = (_tailStart + 1) % _tail.Length; } + } + } // Locked: the reader may be appending while Complete-PkProcess takes it. public string Keep() { lock (_gate) { - if (_text != null) { _kept = _text.ToString(); _text = null; } + if (_text != null) + { + if (_tail != null && _tailCount > 0) + { + int skip = 0; + // Once the ring has wrapped, it can start on the second half of a pair. + if (_total > _text.Length + _tailCount && Char.IsLowSurrogate(_tail[_tailStart])) { skip = 1; } + for (int k = skip; k < _tailCount; k++) { _text.Append(_tail[(_tailStart + k) % _tail.Length]); } + _tailCount = 0; + } + _kept = _text.ToString(); + _dropped = _total - _kept.Length; + _text = null; + } return _kept; } } - public int HeldLength { get { lock (_gate) { return _text != null ? _text.Length : _kept.Length; } } } + public int HeldLength { get { lock (_gate) { return _text != null ? _text.Length + _tailCount : _kept.Length; } } } + public long DroppedChars { get { lock (_gate) { return _dropped; } } } } private readonly Buffer _buffer; private PkStreamReadTask(Buffer buffer) : base(new Func(buffer.Pump), TaskCreationOptions.LongRunning) { _buffer = buffer; } - public static PkStreamReadTask Run(TextReader reader) + public static PkStreamReadTask Run(TextReader reader) { return Run(reader, 0); } + + // maxChars: 0 for no cap. + public static PkStreamReadTask Run(TextReader reader, int maxChars) { - PkStreamReadTask task = new PkStreamReadTask(new Buffer(reader)); + PkStreamReadTask task = new PkStreamReadTask(new Buffer(reader, maxChars)); task.Start(TaskScheduler.Default); return task; } @@ -318,6 +372,10 @@ public sealed class PkStreamReadTask : Task // on; all of it if the stream has already ended. public string Snapshot() { return _buffer.Keep(); } + // How many characters the cap left out of the kept text: 0 with no cap, + // and until the text is kept (the end of the stream or a snapshot). + public long DroppedChars { get { return _buffer.DroppedChars; } } + // Test-only reads: how many characters the buffer holds, and whether the // reader has been closed. public int TestHeldLength { get { return _buffer.HeldLength; } } @@ -812,7 +870,7 @@ public static class PkChildProcessJob # A session that loaded an older copy keeps its compiled types. Say so, # rather than fail later with a missing-method error. Keep this number in # sync with [PkState]::TypesVersion. -if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 9) { +if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 10) { throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one." } @@ -1424,6 +1482,11 @@ function Start-PkProcess { # .PARAMETER OutputEncoding # How to decode its output: a name like 'utf-8', a code page number (65001 # or '65001'), or a [System.Text.Encoding]. Default: the console's code page. + # .PARAMETER MaxOutputChars + # Keep at most this many characters of each output stream (default 0, no + # cap): past it, the first half and the last half, and the result's + # StdoutDroppedChars and StderrDroppedChars say how many were left out. + # The rest is still read, so the program never stalls on a full pipe. # .OUTPUTS # A hashtable @{ Proc; StdoutTask; StderrTask; Stopwatch }. Proc is the # program's Process. Read its output from StdoutTask and StderrTask: Proc's @@ -1433,9 +1496,11 @@ function Start-PkProcess { [string]$FilePath, $Arguments = "", [string]$WorkingDirectory = "", - $OutputEncoding = $null + $OutputEncoding = $null, + [int]$MaxOutputChars = 0 ) - return (Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -CallerState $PSCmdlet.SessionState) + if ($MaxOutputChars -lt 0) { throw ("Start-PkProcess: -MaxOutputChars must be 0 (no cap) or more (got " + $MaxOutputChars + "). It's how many characters of each output stream to keep.") } + return (Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -MaxOutputChars $MaxOutputChars -CallerState $PSCmdlet.SessionState) } function Start-PkProcessCore { @@ -1448,6 +1513,7 @@ function Start-PkProcessCore { $Arguments = "", [string]$WorkingDirectory = "", $OutputEncoding = $null, + [int]$MaxOutputChars = 0, [System.Management.Automation.SessionState]$CallerState ) $encoding = Resolve-PkEncoding -Encoding $OutputEncoding @@ -1520,16 +1586,16 @@ function Start-PkProcessCore { if ($null -ne $started) { return @{ Proc = $started.Process - StdoutTask = [PkStreamReadTask]::Run($started.Stdout) - StderrTask = [PkStreamReadTask]::Run($started.Stderr) + StdoutTask = [PkStreamReadTask]::Run($started.Stdout, $MaxOutputChars) + StderrTask = [PkStreamReadTask]::Run($started.Stderr, $MaxOutputChars) Stopwatch = [System.Diagnostics.Stopwatch]::StartNew() } } $null = [PkChildProcessJob]::Assign($proc) return @{ Proc = $proc - StdoutTask = [PkStreamReadTask]::Run($proc.StandardOutput) - StderrTask = [PkStreamReadTask]::Run($proc.StandardError) + StdoutTask = [PkStreamReadTask]::Run($proc.StandardOutput, $MaxOutputChars) + StderrTask = [PkStreamReadTask]::Run($proc.StandardError, $MaxOutputChars) Stopwatch = [System.Diagnostics.Stopwatch]::StartNew() } } @@ -1671,6 +1737,21 @@ function Get-PkJobTimeLimit { return $limit } +function Get-PkJobMaxOutputChars { + # A job's output cap in characters: its own MaxOutputChars key if it has + # one, else the default. 0 = no cap. Throws on a bad key, as + # Get-PkJobTimeLimit does for TimeoutSeconds. + param([hashtable]$Job, [int]$DefaultMaxOutputChars = 0) + $value = $Job['MaxOutputChars'] + if ($null -eq $value) { return $DefaultMaxOutputChars } + $cap = [long]0 + $parsed = [long]::TryParse(([string]$value).Trim(), [System.Globalization.NumberStyles]::None, [System.Globalization.CultureInfo]::InvariantCulture, [ref]$cap) + if (-not $parsed -or $cap -gt [int]::MaxValue) { + throw ("MaxOutputChars must be a whole number of characters, 0 (no cap) or more (got '" + $value + "').") + } + return [int]$cap +} + function Stop-PkTimedOutJob { # Kills a dispatcher job's program tree once it passes its TimeLimit and # sets TimedOut; the job is then finalized like any other. Only a .Proc @@ -1803,8 +1884,10 @@ function Complete-PkProcess { # .PARAMETER Dispose # Dispose the process object afterwards. # .OUTPUTS - # An object with ExitCode, Stdout, Stderr, StreamTimedOut, StreamError and - # Duration (how long the program ran, from its start to its exit or kill). + # An object with ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, + # Duration (how long the program ran, from its start to its exit or kill), + # and StdoutDroppedChars and StderrDroppedChars (how many characters + # Start-PkProcess's -MaxOutputChars left out of each; 0 with no cap). # Only -Job by position: otherwise `-Dispose $true` would bind $true to # -StreamTimeoutMs (1 ms) and cut the output off without an error. @@ -1823,7 +1906,8 @@ function Complete-PkProcess { if ($StreamTimeoutMs -lt 0) { throw ("Complete-PkProcess: -StreamTimeoutMs must be 0 or more (got " + $StreamTimeoutMs + "). It's how long to wait for the output after the program exits, in milliseconds; 0 doesn't wait.") } - $result = [ordered]@{ ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; Duration = $null } + $result = [ordered]@{ ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null; Duration = $null + StdoutDroppedChars = [long]0; StderrDroppedChars = [long]0 } # A custom starter's job may have no read tasks, and a read can fail. # Neither throws: this runs inside JobFinalize, where a throw stops the # whole dispatcher. @@ -1871,6 +1955,8 @@ function Complete-PkProcess { if ($null -eq $task) { continue } if ($task.Status -eq $ranToCompletion) { $result[$stream] = $task.Result } elseif (-not $task.IsCompleted -and $task -is [PkStreamReadTask]) { $result[$stream] = $task.Snapshot() } + # After the text is taken, which fixes the count. + if ($task -is [PkStreamReadTask]) { $result[$stream + 'DroppedChars'] = $task.DroppedChars } } $result.ExitCode = $Job['Proc'].ExitCode # The program's end, not the output's: before Dispose, which loses it. @@ -1900,12 +1986,16 @@ function Invoke-PkProcessWithTimeout { # Where to run it, as for Start-PkProcess. Default: your current location. # .PARAMETER OutputEncoding # How to decode its output, as for Start-PkProcess. + # .PARAMETER MaxOutputChars + # Keep at most this many characters of each output stream, as for + # Start-PkProcess (default 0, no cap). # .PARAMETER IgnoreAbort # Keep going during an abort, for a cleanup or restore step that has to run # because of it. # .OUTPUTS - # An object with ExitCode, Stdout, Stderr, StreamTimedOut, StreamError and - # Duration (how long the program ran, from its start to its exit or kill). + # Complete-PkProcess's object: ExitCode, Stdout, Stderr, StreamTimedOut, + # StreamError, Duration (how long the program ran, from its start to its + # exit or kill), StdoutDroppedChars and StderrDroppedChars. # .EXAMPLE # $r = Invoke-PkProcessWithTimeout -FilePath git.exe -Arguments 'rev-parse HEAD' -TimeoutMs 10000 param( @@ -1916,13 +2006,15 @@ function Invoke-PkProcessWithTimeout { [int]$StreamTimeoutMs = 15000, [string]$WorkingDirectory = "", $OutputEncoding = $null, + [int]$MaxOutputChars = 0, [switch]$IgnoreAbort ) # Checked before starting anything: 0 or less would kill the program at once. if ($TimeoutMs -lt 1) { throw ("Invoke-PkProcessWithTimeout: -TimeoutMs must be at least 1 (got " + $TimeoutMs + "). It's the program's time limit in milliseconds.") } # Complete-PkProcess makes the same check, but only after the program has run. if ($StreamTimeoutMs -lt 0) { throw ("Invoke-PkProcessWithTimeout: -StreamTimeoutMs must be 0 or more (got " + $StreamTimeoutMs + "). It's how long to wait for the output after the program exits, in milliseconds; 0 doesn't wait.") } - $job = Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -CallerState $PSCmdlet.SessionState + if ($MaxOutputChars -lt 0) { throw ("Invoke-PkProcessWithTimeout: -MaxOutputChars must be 0 (no cap) or more (got " + $MaxOutputChars + "). It's how many characters of each output stream to keep.") } + $job = Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -MaxOutputChars $MaxOutputChars -CallerState $PSCmdlet.SessionState try { # Short slices, so an abort or cancel is noticed within a moment. $waited = [System.Diagnostics.Stopwatch]::StartNew() @@ -1973,8 +2065,9 @@ function Invoke-PkProcessDispatch { # OutputWaitOver, OutputWaitMs. It also sets Group to # the job's group, unless the job already has a Group of its own. The default # starter reads FilePath, Arguments (a string as written, or a list quoted one - # argument per element), WorkingDirectory and OutputEncoding from the job. A - # job's TimeoutSeconds key overrides -TimeoutSeconds. + # argument per element), WorkingDirectory, OutputEncoding and MaxOutputChars + # from the job. A job's TimeoutSeconds key overrides -TimeoutSeconds, and its + # MaxOutputChars key -MaxOutputChars. # .PARAMETER GroupProvider # { } returning the next group, or $null when there are no more, and # outputting nothing else. Return a list as one group with a comma: @@ -2029,6 +2122,10 @@ function Invoke-PkProcessDispatch { # How long the default -JobPollCompleted waits for a job's output after its # program exits, in milliseconds (default 15000); 0 or more, and 0 doesn't # wait. Complete-PkProcess in -JobFinalize uses it too, unless given its own. + # .PARAMETER MaxOutputChars + # For the default starter: keep at most this many characters of each job's + # output streams, as for Start-PkProcess (default 0, no cap). A job's own + # MaxOutputChars key overrides it, and 0 there opts the job out. # .OUTPUTS # An object with GroupResults, Aborted, ForceKilledCount and # JobStarterFailedCount. Each group result has GroupInfo (the group), Completed @@ -2057,7 +2154,8 @@ function Invoke-PkProcessDispatch { [double]$TimeoutSeconds = 0, # Complete-PkProcess's and Invoke-PkProcessBatch's -StreamTimeoutMs # default to the same 15000: keep the three in step. - [int]$StreamTimeoutMs = 15000 + [int]$StreamTimeoutMs = 15000, + [int]$MaxOutputChars = 0 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't # a module, so a callback without .GetNewClosure() sees this frame's @@ -2077,7 +2175,8 @@ function Invoke-PkProcessDispatch { $pkGroupReleaseCheck = $GroupReleaseCheck $pkTimeoutSeconds = $TimeoutSeconds $pkStreamTimeoutMs = $StreamTimeoutMs - Remove-Variable -Name GroupProvider, JobBuilder, JobStarter, JobPollCompleted, JobFinalize, JobAbortCleanup, GroupCompleteCallback, MaxConcurrent, PollCallback, CancelToken, AbortTimeoutSeconds, MaxQueueAhead, GroupReleaseCheck, TimeoutSeconds, StreamTimeoutMs -Scope Local + $pkMaxOutputChars = $MaxOutputChars + Remove-Variable -Name GroupProvider, JobBuilder, JobStarter, JobPollCompleted, JobFinalize, JobAbortCleanup, GroupCompleteCallback, MaxConcurrent, PollCallback, CancelToken, AbortTimeoutSeconds, MaxQueueAhead, GroupReleaseCheck, TimeoutSeconds, StreamTimeoutMs, MaxOutputChars -Scope Local # 0 or less would never start a job (or pull a group) and spin forever. if ($pkMaxConcurrent -isnot [System.Threading.SemaphoreSlim]) { @@ -2089,8 +2188,8 @@ function Invoke-PkProcessDispatch { } if ($pkMaxQueueAhead -lt 1) { throw "Invoke-PkProcessDispatch: -MaxQueueAhead must be at least 1." } # Invoke-PkProcessBatch makes the same -MaxConcurrent, -TimeoutSeconds, - # -AbortTimeoutSeconds and -StreamTimeoutMs checks, so its errors name the - # batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and + # -AbortTimeoutSeconds, -StreamTimeoutMs and -MaxOutputChars checks, so its + # errors name the batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and # acted as no limit. A job's own TimeoutSeconds key follows the same rule # (Get-PkJobTimeLimit). if ([double]::IsNaN($pkTimeoutSeconds) -or [double]::IsInfinity($pkTimeoutSeconds) -or $pkTimeoutSeconds -lt 0) { @@ -2102,6 +2201,9 @@ function Invoke-PkProcessDispatch { if ($pkStreamTimeoutMs -lt 0) { throw ("Invoke-PkProcessDispatch: -StreamTimeoutMs must be 0 or more (got " + $pkStreamTimeoutMs + "). It's how long to wait for a job's output after its program exits, in milliseconds; 0 doesn't wait.") } + if ($pkMaxOutputChars -lt 0) { + throw ("Invoke-PkProcessDispatch: -MaxOutputChars must be 0 (no cap) or more (got " + $pkMaxOutputChars + "). It's how many characters of each job's output streams to keep.") + } # Size the thread pool up front. For a shared budget, its free slots now # are the best measure of its size. @@ -2114,7 +2216,7 @@ function Invoke-PkProcessDispatch { $pkCallerState = $PSCmdlet.SessionState $pkJobStarter = { param($pkJob) - $pkStarted = Start-PkProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory ([string]$pkJob['WorkingDirectory']) -OutputEncoding $pkJob['OutputEncoding'] + $pkStarted = Start-PkProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory ([string]$pkJob['WorkingDirectory']) -OutputEncoding $pkJob['OutputEncoding'] -MaxOutputChars (Get-PkJobMaxOutputChars -Job $pkJob -DefaultMaxOutputChars $pkMaxOutputChars) # Start-PkProcessCore's four keys (it builds them in two places); the # batch's starter copies them too: a new key goes in all four. $pkJob['Proc'] = $pkStarted.Proc; $pkJob['StdoutTask'] = $pkStarted.StdoutTask; $pkJob['StderrTask'] = $pkStarted.StderrTask @@ -2684,10 +2786,12 @@ function New-PkBatchResult { $Process = $null ) $r = [ordered]@{ Command = $Command; ExitCode = $null; Stdout = $null; Stderr = $null; StreamTimedOut = $false; StreamError = $null - StartError = $null; Aborted = [bool]$Aborted; TimedOut = $TimedOut; Duration = $null } + StartError = $null; Aborted = [bool]$Aborted; TimedOut = $TimedOut; Duration = $null + StdoutDroppedChars = [long]0; StderrDroppedChars = [long]0 } if ($null -ne $Completed) { $r.ExitCode = $Completed.ExitCode; $r.Stdout = $Completed.Stdout; $r.Stderr = $Completed.Stderr $r.StreamTimedOut = $Completed.StreamTimedOut; $r.StreamError = $Completed.StreamError; $r.Duration = $Completed.Duration + $r.StdoutDroppedChars = $Completed.StdoutDroppedChars; $r.StderrDroppedChars = $Completed.StderrDroppedChars } elseif ($null -ne $Process) { $r.Duration = Get-PkProcessDuration -Process $Process } @@ -2742,10 +2846,15 @@ function Invoke-PkProcessBatch { # How long to wait for a program's output after it exits, in milliseconds # (default 15000); 0 or more, and 0 doesn't wait. Past it, the result has # StreamTimedOut = $true and the output that arrived. + # .PARAMETER MaxOutputChars + # Keep at most this many characters of each program's output streams, as + # for Start-PkProcess (default 0, no cap). A command's own MaxOutputChars + # key overrides it, and 0 there opts the command out. # .OUTPUTS # An object with Results, Aborted and ForceKilledCount. Each result has Command, # ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, StartError, Aborted, - # TimedOut and Duration, in the order the programs finished. + # TimedOut, Duration, StdoutDroppedChars and StderrDroppedChars, in the order + # the programs finished. # .EXAMPLE # $commands = foreach ($f in Get-ChildItem *.png) { @{ FilePath = 'cjxl.exe'; Arguments = @($f.FullName, "$($f.FullName).jxl") } } # $batch = Invoke-PkRun { Invoke-PkProcessBatch -Commands $commands -MaxConcurrent 6 } @@ -2764,7 +2873,8 @@ function Invoke-PkProcessBatch { $OutputEncoding = $null, # Complete-PkProcess's and Invoke-PkProcessDispatch's -StreamTimeoutMs # default to the same 15000: keep the three in step. - [int]$StreamTimeoutMs = 15000 + [int]$StreamTimeoutMs = 15000, + [int]$MaxOutputChars = 0 ) # pk-prefixed, parameters included: inside a worker runspace the kit isn't # a module, so a callback without .GetNewClosure() sees this frame's @@ -2776,11 +2886,12 @@ function Invoke-PkProcessBatch { $pkTimeoutSeconds = $TimeoutSeconds $pkOutputEncoding = $OutputEncoding $pkStreamTimeoutMs = $StreamTimeoutMs - Remove-Variable -Name Commands, MaxConcurrent, OnComplete, AbortTimeoutSeconds, TimeoutSeconds, OutputEncoding, StreamTimeoutMs -Scope Local + $pkMaxOutputChars = $MaxOutputChars + Remove-Variable -Name Commands, MaxConcurrent, OnComplete, AbortTimeoutSeconds, TimeoutSeconds, OutputEncoding, StreamTimeoutMs, MaxOutputChars -Scope Local # Checked once here, so a bad name fails the call instead of every command. $null = Resolve-PkEncoding -Encoding $pkOutputEncoding - # Invoke-PkProcessDispatch makes the same four checks; these name the + # Invoke-PkProcessDispatch makes the same five checks; these name the # batch. Keep the two in step. Get-PkJobTimeLimit checks a job's own # TimeoutSeconds key by the same rule. if ($pkMaxConcurrent -lt 1) { throw ("Invoke-PkProcessBatch: -MaxConcurrent must be at least 1 (got " + $pkMaxConcurrent + ").") } @@ -2793,6 +2904,9 @@ function Invoke-PkProcessBatch { if ($pkStreamTimeoutMs -lt 0) { throw ("Invoke-PkProcessBatch: -StreamTimeoutMs must be 0 or more (got " + $pkStreamTimeoutMs + "). It's how long to wait for a program's output after it exits, in milliseconds; 0 doesn't wait.") } + if ($pkMaxOutputChars -lt 0) { + throw ("Invoke-PkProcessBatch: -MaxOutputChars must be 0 (no cap) or more (got " + $pkMaxOutputChars + "). It's how many characters of each program's output streams to keep.") + } $pkResults = [System.Collections.Generic.List[object]]::new() $pkState = @{ Given = $false } @@ -2801,6 +2915,7 @@ function Invoke-PkProcessBatch { $pkNewBatchResult = ${function:New-PkBatchResult} $pkStartProcessCore = ${function:Start-PkProcessCore} $pkGetJobTimeLimit = ${function:Get-PkJobTimeLimit} + $pkGetJobMaxOutputChars = ${function:Get-PkJobMaxOutputChars} $pkGetStartFailureCause = ${function:Get-PkStartFailureCause} $pkWriteCallbackLoopWarning = ${function:Write-PkCallbackLoopWarning} # Invoke-PkCallback's -PkContext: callbacks that used continue/break are @@ -2842,11 +2957,12 @@ function Invoke-PkProcessBatch { # starter below, so the command would get no result. try { $null = & $pkGetJobTimeLimit -Job $pkC + $pkCap = & $pkGetJobMaxOutputChars -Job $pkC -DefaultMaxOutputChars $pkMaxOutputChars } catch { & $pkReportStartFailure $pkC $_ continue } - $pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = $pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; Command = $pkC }) + $pkJobs.Add(@{ FilePath = $pkC['FilePath']; Arguments = $pkC['Arguments']; WorkingDirectory = [string]$pkC['WorkingDirectory']; OutputEncoding = $pkEncoding; TimeoutSeconds = $pkC['TimeoutSeconds']; MaxOutputChars = $pkCap; Command = $pkC }) } return $pkJobs }.GetNewClosure() @@ -2854,7 +2970,7 @@ function Invoke-PkProcessBatch { $pkJobStarter = { param($pkJob) try { - $pkStarted = & $pkStartProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory $pkJob['WorkingDirectory'] -OutputEncoding $pkJob['OutputEncoding'] + $pkStarted = & $pkStartProcessCore -CallerState $pkCallerState -FilePath $pkJob['FilePath'] -Arguments $pkJob['Arguments'] -WorkingDirectory $pkJob['WorkingDirectory'] -OutputEncoding $pkJob['OutputEncoding'] -MaxOutputChars $pkJob['MaxOutputChars'] } catch { $pkJob['StartFailureReported'] = $true & $pkReportStartFailure $pkJob['Command'] $_ @@ -2915,7 +3031,7 @@ function Get-PkWorkerSafeFunctionNames { return @('Write-PkMessage', 'Test-PkAbortSignal', 'Test-PkCancelRequested', 'Initialize-PkThreadPool', 'Stop-PkProcessTree', 'Stop-PkProcessTreeCore', 'Split-PkTaskkillArguments', 'Stop-PkJobProcesses', 'Get-PkProcessDuration', 'Resolve-PkEncoding', 'Start-PkProcess', 'Wait-PkProcess', 'Wait-PkRunningJobs', 'Complete-PkProcess', 'Invoke-PkProcessWithTimeout', 'Invoke-PkProcessDispatch', 'Invoke-PkProcessDispatchCore', 'Invoke-PkProcessBatch', - 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause', 'Resolve-PkArguments', 'Invoke-PkCallback', 'Write-PkCallbackLoopWarning', 'ConvertTo-PkArgumentStringCore') + 'Register-PkCancellationSource', 'Unregister-PkCancellationSource', 'New-PkBatchResult', 'Get-PkJobMaxOutputChars', 'ConvertTo-PkArgumentString', 'Get-PkForceStopReason', 'Resolve-PkFileSystemPath', 'Add-PkDispatchGroupJobs', 'Start-PkProcessCore', 'Get-PkJobTimeLimit', 'Stop-PkTimedOutJob', 'Get-PkStartFailureCause', 'Resolve-PkArguments', 'Invoke-PkCallback', 'Write-PkCallbackLoopWarning', 'ConvertTo-PkArgumentStringCore') } function Get-PkHandlerParamCount { From bb56085aeb2054bddefe028071007f28e7d4b5e2 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:57:53 -0500 Subject: [PATCH 4/5] Document -MaxOutputChars Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 489e8e0..47036ae 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Only the list form gets this quoting. A string, including one built with `Conver Programs run in your current location (`Get-Location`) unless a command says otherwise, and a relative `WorkingDirectory`, or a `FilePath` with a folder in it like `.\tools\cjxl.exe`, is relative to that location too. Drives PowerShell knows, like `~` or one made with `New-PSDrive`, work in both. A bare name like `cjxl.exe` is found the way Windows finds programs, mostly on `PATH`. Only an `.exe` is found without its extension (Windows doesn't use `PATHEXT` here), so a `.cmd` wrapper such as `npm` has to be written out: `npm.cmd`. A `.bat` or `.cmd` program can't run in a network-share (`\\server\share`) folder: `cmd.exe` falls back to `C:\Windows` there, so give such commands a local `WorkingDirectory`. -`Invoke-PkProcessBatch` returns an object with `Results`, `Aborted` and `ForceKilledCount`. Each result is an object with `Command`, `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut`, `StreamError`, `StartError`, `Aborted`, `TimedOut` and `Duration`, every field always present, so `$batch.Results | Format-Table` reads well: +`Invoke-PkProcessBatch` returns an object with `Results`, `Aborted` and `ForceKilledCount`. Each result is an object with `Command`, `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut`, `StreamError`, `StartError`, `Aborted`, `TimedOut`, `Duration`, `StdoutDroppedChars` and `StderrDroppedChars`, every field always present, so `$batch.Results | Format-Table` reads well: - `Command` is your original command hashtable, so extra keys like `Source` above come back with it. - `ExitCode` is whatever the program returned. The kit doesn't treat a non-zero code as a failure; that's for you to decide. @@ -144,6 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. +- `StdoutDroppedChars` and `StderrDroppedChars` count the characters `-MaxOutputChars` left out of each stream; 0 when nothing was, or with no cap. - `StreamTimedOut` means the output read gave up 15 seconds after the program exited (`-StreamTimeoutMs` changes that, in milliseconds; 0 doesn't wait), on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. While a program's output is held open, its command keeps its slot, but the other commands carry on: they keep starting, finishing and being killed at their time limits, and an abort is acted on at once. If an abort's wait for running programs ends while one is still waiting for its output, it gets what has arrived, with `StreamTimedOut`, rather than counting as force-stopped. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`. @@ -152,6 +153,8 @@ Programs run in your current location (`Get-Location`) unless a command says oth `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`. +`-MaxOutputChars 100000` keeps at most 100,000 characters of each program's stdout and stderr. Past that, the first half and the last half are kept, so the start and the end of a long log are both there, and `StdoutDroppedChars` / `StderrDroppedChars` say how many characters were left out. The rest is still read, so the program never stalls on a full pipe. The default, 0, keeps everything. A command's own `MaxOutputChars` key overrides it; 0 opts that command out. `Start-PkProcess`, `Invoke-PkProcessWithTimeout` and the dispatcher take it too. + Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`, or the same as a string, as read from a CSV or config file) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. @@ -254,7 +257,7 @@ All of these callbacks run on whichever thread called the dispatcher: the main t - A group can be any value: a hashtable, a path, a name. Each job gets its group as `$job.Group`. - Each `-GroupProvider` call outputs exactly one group, or `$null` to end, and nothing else: `return $group`, `return ,@('a', 'b', 'c')` for a list as one group (the comma keeps a one-item list a list), or `return $null`. Anything else it lets out (an `ArrayList.Add` index, say) becomes part of the group, so discard it with `$null = ...`. Output that ends in `$null` is taken as the end, with a warning, so a list group can't end in `$null`. - The dispatcher keeps its state on the job hashtables. Don't use these names for your own job fields: `GroupKey`, `SlotReleased`, `Proc`, `StdoutTask`, `StderrTask`, `Stopwatch`, `TimeLimit`, `TimedOut`, `StartFailureReported`, `OutputWait`, `OutputWaitOver`, `OutputWaitMs`. (`Group` is yours if you set it yourself.) -- `-TimeoutSeconds 600` kills a job's program, and anything it started, if it's still running after 10 minutes. The default, 0, means no limit, and it can be fractional. A job's own `TimeoutSeconds` key overrides it; 0 opts the job out, and a bad value is a failed start. The killed job still goes through `-JobFinalize`, where `$job['TimedOut']` is `$true`, and other jobs carry on. The limit stays in force while an abort waits for running jobs. A retry gets a fresh clock. A job whose `.Proc` isn't a process isn't timed: `-JobPollCompleted` is the place for its limit. +- `-TimeoutSeconds 600` kills a job's program, and anything it started, if it's still running after 10 minutes. The default, 0, means no limit, and it can be fractional. A job's own `TimeoutSeconds` key overrides it; 0 opts the job out, and a bad value is a failed start. `-MaxOutputChars` works the same way, with a job's `MaxOutputChars` key, for the default starter. The killed job still goes through `-JobFinalize`, where `$job['TimedOut']` is `$true`, and other jobs carry on. The limit stays in force while an abort waits for running jobs. A retry gets a fresh clock. A job whose `.Proc` isn't a process isn't timed: `-JobPollCompleted` is the place for its limit. - To retry a job or add follow-up work, enqueue job hashtables onto `$pending` inside `-JobFinalize`. They join the same group, and the group isn't complete until they've all finished. A retry can start within milliseconds of the failure, so cap the number of retries, and don't count on the gap for something like a locked file to clear. - Supply `-JobStarter { param($job) ... }` to start jobs yourself. It must set `$job.Proc` and should return the job. A starter that is just `Start-PkProcess ...` works too: its result is folded into your job. Throw to report a job that couldn't start. - Whatever your callbacks output is thrown away, apart from the return values listed here. Where a callback returns something, output only that: `-JobPollCompleted` and `-GroupReleaseCheck` go by their last value and `-JobBuilder` ignores anything that isn't a job hashtable, but a stray value from `-GroupProvider` becomes part of the group. @@ -274,7 +277,7 @@ To leave any callback early, use `return`. PowerShell sends a `continue` or `bre | Function | Use | |---|---| -| `Invoke-PkProcessWithTimeout -FilePath -Arguments -TimeoutMs` | Run one program and wait, with a hard time limit (default 60s). Returns an object with `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut`, `StreamError` and `Duration`. If it overruns, or the run is aborted, the program is killed and this **throws**; add `-IgnoreAbort` for a cleanup step that must finish anyway. | +| `Invoke-PkProcessWithTimeout -FilePath -Arguments -TimeoutMs` | Run one program and wait, with a hard time limit (default 60s). Returns an object with `ExitCode`, `Stdout`, `Stderr`, `StreamTimedOut`, `StreamError`, `Duration`, `StdoutDroppedChars` and `StderrDroppedChars`. If it overruns, or the run is aborted, the program is killed and this **throws**; add `-IgnoreAbort` for a cleanup step that must finish anyway. | | `ConvertTo-PkArgumentString @(args)` | Join a list of arguments into one correctly quoted command line. `-Arguments` / `Arguments` also take the list directly, and then also quote it for `cmd.exe` when the program is a `.bat` or `.cmd`. | | `Start-PkProcess -FilePath -Arguments` | Start one hidden, with output captured in the background. Returns `@{ Proc; StdoutTask; StderrTask; Stopwatch }`; throws if it can't start. | | `Wait-PkProcess -Process -TimeoutMs` | Wait for exit with a timeout that can't overrun. Returns `$true` if it exited. | @@ -333,7 +336,7 @@ Other pieces: `Request-PkAbort` triggers the same abort from code. `Test-PkAbort - One run at a time per process. The abort state is shared. A nested `Invoke-PkRun` just runs inside the outer one. - A worker stuck inside a long .NET call can't be interrupted. Force-stop stops waiting for it and returns. The worker is NOT killed: once that call returns it can carry on to the end of its script. Before any step that can't be undone (deleting a source file, say), have the worker check `Test-PkCancelRequested` and skip the step if it's true. - Programs run hidden, and there's no way to send them input. A program that stops to ask a question will sit there until it's killed. -- A program's output is collected in memory and handed over when it exits; there's no live line-by-line output. A program that prints gigabytes will use that much memory. A batch keeps every result, output included, until it returns, which adds up over thousands of chatty programs. If you only need the output in `-OnComplete`, drop it there with `$result.Stdout = $null; $result.Stderr = $null` and it's freed as the batch goes. +- A program's output is collected in memory and handed over when it exits; there's no live line-by-line output. A program that prints gigabytes will use that much memory, unless `-MaxOutputChars` caps what's kept. A batch keeps every result, output included, until it returns, which adds up over thousands of chatty programs. If you only need the output in `-OnComplete`, drop it there with `$result.Stdout = $null; $result.Stderr = $null` and it's freed as the batch goes. ## Testing From 3be3a0f528ff255212d1624df3ffbdfaba257d73 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:09:30 -0500 Subject: [PATCH 5/5] Grow the cap's tail with the output, copy in blocks, never end a snapshot in half a pair Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 15 +++++++++ ParallelKit.ps1 | 74 ++++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index ef422b2..a481b70 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4856,6 +4856,14 @@ $beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap 10 Test-Assert ($beR.Stdout -ceq "01234HIJ`r`n" -and $beR.StdoutDroppedChars -eq 12) ("past -MaxOutputChars 10, the first 5 and the last 5 characters are kept, and 12 counted as dropped (" + (Format-HarnessCapRun $beR) + ")") $beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap 22 Test-Assert ($beR.Stdout -ceq "0123456789ABCDEFGHIJ`r`n" -and $beR.StdoutDroppedChars -eq 0) ("output exactly at the cap comes back whole, nothing dropped (" + (Format-HarnessCapRun $beR) + ")") +# The largest cap costs nothing until output reaches it: the ring grows +# with the output, so this doesn't try to allocate a billion characters. +$beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap ([int]::MaxValue) +Test-Assert ($beR.Stdout -ceq "0123456789ABCDEFGHIJ`r`n" -and $beR.StdoutDroppedChars -eq 0) ("-MaxOutputChars [int]::MaxValue keeps short output whole (" + (Format-HarnessCapRun $beR) + ")") +# The Process.Start fallback reads through the same capped reader. +[PkChildProcessJob]::TestUseProcessStart = $true +try { $beR = Get-HarnessCapRun -Arguments '/d /c echo 0123456789ABCDEFGHIJ' -Cap 10 } finally { [PkChildProcessJob]::TestUseProcessStart = $false } +Test-Assert ($beR.Stdout -ceq "01234HIJ`r`n" -and $beR.StdoutDroppedChars -eq 12) ("the Process.Start fallback is capped the same way (" + (Format-HarnessCapRun $beR) + ")") $beR = Get-HarnessCapRun -Arguments '/d /c 1>&2 echo 0123456789ABCDEFGHIJ' -Cap 10 Test-Assert ($beR.Stderr -ceq "01234HIJ`r`n" -and $beR.StderrDroppedChars -eq 12 -and $beR.StdoutDroppedChars -eq 0) ("stderr is capped the same way, on its own ('" + ([string]$beR.Stderr -replace "`r", '\r' -replace "`n", '\n') + "', dropped " + $beR.StderrDroppedChars + ")") @@ -4886,6 +4894,13 @@ $beWholeLen = -1; if ($null -ne $beWhole) { $beWholeLen = ([string]$beWhole.Stdo $beOk = ($null -ne $beR -and ([string]$beR.Stdout).Length -eq 1000 -and $beR.Stdout.StartsWith("line1`r`nline2`r`n") -and $beR.Stdout.EndsWith("line20000`r`n") -and ($beR.StdoutDroppedChars + 1000) -eq $beWholeLen) Test-Assert ($beOk -and $beHeldMax -le 1000) ("20,000 lines under a 1,000-character cap: 1,000 kept, from line1 to line20000, the dropped count adds up to the uncapped " + $beWholeLen + ", and at most " + $beHeldMax + " held while it ran" + $(if ($beErr) { " (error: " + $beErr + ")" })) +# A cap bigger than one read (4,096 characters), so the ring wraps part-way +# through reads: the kept text is exactly the uncapped text's first and last +# 10,000 characters. +$beR = $null; try { $beR = Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments $beLoop -MaxOutputChars 20000 -TimeoutMs 120000 } catch { } +$beExpect = $null; if ($null -ne $beWhole -and $beWholeLen -gt 20000) { $beExpect = $beWhole.Stdout.Substring(0, 10000) + $beWhole.Stdout.Substring($beWholeLen - 10000) } +Test-Assert ($null -ne $beR -and $null -ne $beExpect -and $beR.Stdout -ceq $beExpect -and $beR.StdoutDroppedChars -eq ($beWholeLen - 20000)) ("a 20,000-character cap keeps exactly the uncapped output's first and last 10,000 characters (" + $(if ($beR) { ([string]$beR.Stdout).Length.ToString() + " kept, dropped " + $beR.StdoutDroppedChars } else { 'no result' }) + ")") + # A stream timeout takes the start and the end of what had arrived. try { $beJob = $null; $beR = $null; $beErr = $null diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 1b67618..3cf31de 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -253,8 +253,9 @@ public sealed class PkStartedProcess // but drops what it reads, which nobody can get any more. The task's result // is that snapshot, or the whole text if none was taken. The reader is // closed at the end of the stream. The reader's decoder carries a -// character split across two reads over to the next, so a snapshot never -// ends in half a character. Each read runs on a thread of its own +// character split across two reads over to the next, and a snapshot leaves +// out a surrogate pair's first half whose second half hasn't arrived, so a +// snapshot never ends in half a character. Each read runs on a thread of its own // (LongRunning), not a thread-pool thread: a read blocks until every holder // of the stream's write end has let go (at the latest when PowerShell exits: // they're in the Job Object), and leftover programs holding their output @@ -275,16 +276,18 @@ public sealed class PkStreamReadTask : Task private string _kept; private readonly object _gate = new object(); public volatile bool ReaderClosed; - // The cap's tail: a ring of the last chars read. Null with no cap. - private readonly int _headCap; + // The cap's tail: the last chars read, a ring once it's full. It + // grows with the output, so a cap costs nothing until output reaches + // it. _tailCap is 0 with no cap. + private readonly int _headCap, _tailCap; private bool _headFull; - private readonly char[] _tail; + private char[] _tail; private int _tailStart, _tailCount; private long _total, _dropped; public Buffer(TextReader reader, int maxChars) { _reader = reader; - if (maxChars > 0) { _headCap = maxChars / 2; _tail = new char[maxChars - _headCap]; } + if (maxChars > 0) { _headCap = maxChars / 2; _tailCap = maxChars - _headCap; } } public string Pump() { @@ -296,7 +299,7 @@ public sealed class PkStreamReadTask : Task { lock (_gate) { if (_text != null) { Append(chunk, n); } } } - return Keep(); + return Keep(false); } finally { @@ -309,7 +312,7 @@ public sealed class PkStreamReadTask : Task private void Append(char[] chunk, int n) { _total += n; - if (_tail == null) { _text.Append(chunk, 0, n); return; } + if (_tailCap == 0) { _text.Append(chunk, 0, n); return; } int i = 0; if (!_headFull) { @@ -323,27 +326,55 @@ public sealed class PkStreamReadTask : Task _text.Append(chunk, 0, take); i = take; } - for (; i < n; i++) + if (i < n) { AppendTail(chunk, i, n - i); } + } + // Under _gate. Until the ring is full it's filled in order from 0; + // after, _tailStart is the oldest char. + private void AppendTail(char[] src, int offset, int count) + { + if (_tailCount < _tailCap) { - if (_tailCount < _tail.Length) { _tail[(_tailStart + _tailCount) % _tail.Length] = chunk[i]; _tailCount++; } - else { _tail[_tailStart] = chunk[i]; _tailStart = (_tailStart + 1) % _tail.Length; } + int take = Math.Min(_tailCap - _tailCount, count); + int need = _tailCount + take; + if (_tail == null || _tail.Length < need) + { + long size = _tail == null ? 4096 : _tail.Length; + while (size < need) { size *= 2; } + Array.Resize(ref _tail, (int)Math.Min(size, _tailCap)); + } + Array.Copy(src, offset, _tail, _tailCount, take); + _tailCount += take; offset += take; count -= take; } - } - // Locked: the reader may be appending while Complete-PkProcess takes it. - public string Keep() + if (count == 0) { return; } + if (count >= _tailCap) + { + Array.Copy(src, offset + count - _tailCap, _tail, 0, _tailCap); + _tailStart = 0; + return; + } + int first = Math.Min(count, _tailCap - _tailStart); + Array.Copy(src, offset, _tail, _tailStart, first); + Array.Copy(src, offset + first, _tail, 0, count - first); + _tailStart = (_tailStart + count) % _tailCap; + } + // Locked: the reader may be appending while Complete-PkProcess takes + // it. A snapshot (the stream still open) leaves out a last char that + // is the first half of a pair whose second half hasn't been read. + public string Keep(bool snapshot) { lock (_gate) { if (_text != null) { - if (_tail != null && _tailCount > 0) + if (_tailCount > 0) { int skip = 0; // Once the ring has wrapped, it can start on the second half of a pair. if (_total > _text.Length + _tailCount && Char.IsLowSurrogate(_tail[_tailStart])) { skip = 1; } - for (int k = skip; k < _tailCount; k++) { _text.Append(_tail[(_tailStart + k) % _tail.Length]); } - _tailCount = 0; + for (int k = skip; k < _tailCount; k++) { _text.Append(_tail[(_tailStart + k) % _tailCap]); } + _tail = null; _tailCount = 0; } + if (snapshot && _text.Length > 0 && Char.IsHighSurrogate(_text[_text.Length - 1])) { _text.Length--; } _kept = _text.ToString(); _dropped = _total - _kept.Length; _text = null; @@ -370,7 +401,7 @@ public sealed class PkStreamReadTask : Task // The text that has arrived so far, kept as the stream's text from then // on; all of it if the stream has already ended. - public string Snapshot() { return _buffer.Keep(); } + public string Snapshot() { return _buffer.Keep(true); } // How many characters the cap left out of the kept text: 0 with no cap, // and until the text is kept (the end of the stream or a snapshot). @@ -1499,6 +1530,8 @@ function Start-PkProcess { $OutputEncoding = $null, [int]$MaxOutputChars = 0 ) + # Invoke-PkProcessWithTimeout, Invoke-PkProcessBatch and Invoke-PkProcessDispatch + # make the same check, naming themselves: keep the four in step. if ($MaxOutputChars -lt 0) { throw ("Start-PkProcess: -MaxOutputChars must be 0 (no cap) or more (got " + $MaxOutputChars + "). It's how many characters of each output stream to keep.") } return (Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -MaxOutputChars $MaxOutputChars -CallerState $PSCmdlet.SessionState) } @@ -2013,6 +2046,8 @@ function Invoke-PkProcessWithTimeout { if ($TimeoutMs -lt 1) { throw ("Invoke-PkProcessWithTimeout: -TimeoutMs must be at least 1 (got " + $TimeoutMs + "). It's the program's time limit in milliseconds.") } # Complete-PkProcess makes the same check, but only after the program has run. if ($StreamTimeoutMs -lt 0) { throw ("Invoke-PkProcessWithTimeout: -StreamTimeoutMs must be 0 or more (got " + $StreamTimeoutMs + "). It's how long to wait for the output after the program exits, in milliseconds; 0 doesn't wait.") } + # Start-PkProcess, Invoke-PkProcessBatch and Invoke-PkProcessDispatch make the + # same check: keep the four in step. if ($MaxOutputChars -lt 0) { throw ("Invoke-PkProcessWithTimeout: -MaxOutputChars must be 0 (no cap) or more (got " + $MaxOutputChars + "). It's how many characters of each output stream to keep.") } $job = Start-PkProcessCore -FilePath $FilePath -Arguments $Arguments -WorkingDirectory $WorkingDirectory -OutputEncoding $OutputEncoding -MaxOutputChars $MaxOutputChars -CallerState $PSCmdlet.SessionState try { @@ -2189,7 +2224,8 @@ function Invoke-PkProcessDispatch { if ($pkMaxQueueAhead -lt 1) { throw "Invoke-PkProcessDispatch: -MaxQueueAhead must be at least 1." } # Invoke-PkProcessBatch makes the same -MaxConcurrent, -TimeoutSeconds, # -AbortTimeoutSeconds, -StreamTimeoutMs and -MaxOutputChars checks, so its - # errors name the batch: keep the two in step. NaN and +Infinity pass a `-lt 0` test and + # errors name the batch: keep the two in step. Start-PkProcess and + # Invoke-PkProcessWithTimeout check -MaxOutputChars the same way. NaN and +Infinity pass a `-lt 0` test and # acted as no limit. A job's own TimeoutSeconds key follows the same rule # (Get-PkJobTimeLimit). if ([double]::IsNaN($pkTimeoutSeconds) -or [double]::IsInfinity($pkTimeoutSeconds) -or $pkTimeoutSeconds -lt 0) {