diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index e024656..aa1074c 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4546,6 +4546,71 @@ try { } Write-Host "" +Write-Host "=== BA: after a stream timeout, the read drains without keeping the text ===" -ForegroundColor Cyan +# The read of a held stream goes on to its end, but nobody can get what it +# reads after the result was handed over, so it isn't kept. The reader is +# closed at the end of every stream, rather than at the next garbage +# collection. Test-only reads that a build without them lacks count as a +# failure, not an error. +Reset-Harness +function Get-HarnessReadState($Task, [string]$Name) { + try { return $Task.$Name } catch { return 'missing' } # an older build has no such test-only read +} +function Stop-HarnessPingsOf([int]$ParentId) { + foreach ($p in @(Get-CimInstance Win32_Process -Filter ("ParentProcessId=" + $ParentId + " AND Name='PING.EXE'"))) { Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue } +} +# A ping that writes a line a second, holding the output after cmd exits. +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo first& start /b ping -t 127.0.0.1' +try { + $null = $s['Proc'].WaitForExit(10000) + $r = Complete-PkProcess $s -StreamTimeoutMs 1500 + $baHeldAtResult = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength' + Start-Sleep -Seconds 3 + $baHeldLater = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength' + $baStillReading = -not $s['StdoutTask'].IsCompleted + $baAgain = Complete-PkProcess $s -StreamTimeoutMs 0 + $baLength = ([string]$r.Stdout).Length + Test-Assert ($baAgain.Stdout -ceq $r.Stdout) ("a second Complete-PkProcess on the same job returns the same text (" + ([string]$baAgain.Stdout).Length + " characters, first " + $baLength + ")") + Test-Assert ($r.StreamTimedOut -and $r.Stdout -match 'first' -and $baStillReading -and $baHeldAtResult -eq $baLength -and $baHeldLater -eq $baLength) ("a program that keeps writing after the result: the buffer stays at the result's " + $baLength + " characters (at the result " + $baHeldAtResult + ", 3 s later " + $baHeldLater + ", still reading " + $baStillReading + ")") +} finally { Stop-HarnessPingsOf -ParentId $s['Proc'].Id } +$baEnded = $s['StdoutTask'].Wait(10000) -and $s['StderrTask'].Wait(10000) +$baFinal = $null; if ($baEnded) { $baFinal = $s['StdoutTask'].Result } +Test-Assert ($baEnded -and $baFinal -eq $r.Stdout -and ([string]$baFinal).Length -eq $baLength) ("once the writer is killed the read ends, and its result is the text handed over, not what came after (" + $baEnded + ", " + ([string]$baFinal).Length + " characters)") +Test-Assert ((Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed') -eq $true) "a held stream's reader is closed when the stream ends" +$s['Proc'].Dispose() + +foreach ($baPath in 'created in the job', 'Process.Start fallback') { + [PkChildProcessJob]::TestUseProcessStart = ($baPath -eq 'Process.Start fallback') + try { + $s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo out& echo err 1>&2' + $r = Complete-PkProcess $s -Dispose + } finally { [PkChildProcessJob]::TestUseProcessStart = $false } + $baOut = Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed' + $baErr = Get-HarnessReadState $s['StderrTask'] 'TestReaderClosed' + Test-Assert ($r.Stdout -match 'out' -and $r.Stderr -match 'err' -and $baOut -eq $true -and $baErr -eq $true) ($baPath + ": both readers are closed once a program's output ends (stdout " + $baOut + ", stderr " + $baErr + ")") +} + +# One read fails while the other stream is still held: the failure is +# reported, not just the timeout. Task.WaitAll with a timeout throws only once +# every task has finished, so it didn't throw here. +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0' +$null = $s['Proc'].WaitForExit(10000) +$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000) +$baFailed = [System.Threading.Tasks.TaskCompletionSource[string]]::new() +$baFailed.SetException([System.IO.IOException]::new('Test: the read failed.')) +$baHeld = [System.Threading.Tasks.TaskCompletionSource[string]]::new() +$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $baHeld.Task } -StreamTimeoutMs 300 -Dispose +$baHeld.SetResult('') +Test-Assert ($r.StreamTimedOut -and $r.StreamError -match 'the read failed' -and $null -eq $r.Stdout) ("a read that failed while the other stream timed out is reported in StreamError (StreamTimedOut " + $r.StreamTimedOut + ", StreamError '" + $r.StreamError + "')") +# And when the other stream ended normally: the read's own reason, not +# WaitAll's "One or more errors occurred". +$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0' +$null = $s['Proc'].WaitForExit(10000) +$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000) +$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $s['StderrTask'] } -StreamTimeoutMs 300 -Dispose +Test-Assert (-not $r.StreamTimedOut -and $r.StreamError -eq 'Test: the read failed.' -and $null -eq $r.Stdout) ("a failed read next to one that ended gives the read's own reason (StreamError '" + $r.StreamError + "')") +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 675310b..fb5ca71 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -164,7 +164,7 @@ public static class PkState // Bump this, and the check after the type definitions, whenever any C# // in this file changes: a session keeps the types it compiled first. - public const int TypesVersion = 8; + public const int TypesVersion = 9; // Locked: Register-/Unregister-PkCancellationSource are callable from // worker runspaces, and List is not safe for concurrent writers. @@ -247,8 +247,12 @@ public sealed class PkStartedProcess // Reads one of a program's output streams to its end, in chunks, so what has // arrived can be taken at any time (Snapshot): a program the program started -// can hold the stream open long after it exits. The task's result is the -// whole text, at the end of the stream. The reader's decoder carries a +// can hold the stream open long after it exits. The first snapshot taken +// before the end is the stream's text from then on: the read carries on to +// the end, so the program holding the stream never blocks on a full pipe, +// but drops what it reads, which nobody can get any more. The task's result +// is that snapshot, or the whole text if none was taken. The reader is +// closed at the end of the stream. The reader's decoder carries a // character split across two reads over to the next, so a snapshot never // ends in half a character. Each read runs on a thread of its own // (LongRunning), not a thread-pool thread: a read blocks until every holder @@ -261,23 +265,43 @@ public sealed class PkStreamReadTask : Task private sealed class Buffer { private readonly TextReader _reader; + // Null once the text is kept: at a snapshot or at the end of the + // stream, whichever comes first. The builder goes then, so the text + // isn't held twice. private StringBuilder _text = new StringBuilder(); - private string _final; + private string _kept; private readonly object _gate = new object(); + public volatile bool ReaderClosed; public Buffer(TextReader reader) { _reader = reader; } public string Pump() { - char[] chunk = new char[4096]; - int n; - while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0) + try + { + char[] chunk = new char[4096]; + int n; + while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0) + { + lock (_gate) { if (_text != null) { _text.Append(chunk, 0, n); } } + } + return Keep(); + } + finally { - lock (_gate) { _text.Append(chunk, 0, n); } + // Closes the pipe now rather than at the next garbage collection. + try { _reader.Dispose(); ReaderClosed = true; } + catch (Exception) { } // a close that fails must not hide the read's own result or error } - // The builder goes once the text is final, so it isn't held twice. - lock (_gate) { _final = _text.ToString(); _text = null; return _final; } } // Locked: the reader may be appending while Complete-PkProcess takes it. - public string Snapshot() { lock (_gate) { return _final != null ? _final : _text.ToString(); } } + public string Keep() + { + lock (_gate) + { + if (_text != null) { _kept = _text.ToString(); _text = null; } + return _kept; + } + } + public int HeldLength { get { lock (_gate) { return _text != null ? _text.Length : _kept.Length; } } } } private readonly Buffer _buffer; @@ -290,8 +314,14 @@ public sealed class PkStreamReadTask : Task return task; } - // The text that has arrived so far; all of it once the task has finished. - public string Snapshot() { return _buffer.Snapshot(); } + // The text that has arrived so far, kept as the stream's text from then + // on; all of it if the stream has already ended. + public string Snapshot() { return _buffer.Keep(); } + + // Test-only reads: how many characters the buffer holds, and whether the + // reader has been closed. + public int TestHeldLength { get { return _buffer.HeldLength; } } + public bool TestReaderClosed { get { return _buffer.ReaderClosed; } } } // A Windows Job Object with KILL_ON_JOB_CLOSE: every child process assigned @@ -782,7 +812,7 @@ public static class PkChildProcessJob # A session that loaded an older copy keeps its compiled types. Say so, # rather than fail later with a missing-method error. Keep this number in # sync with [PkState]::TypesVersion. -if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 8) { +if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 9) { throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one." } @@ -1753,7 +1783,10 @@ function Complete-PkProcess { # that limit was hit. The stream it gave up on comes back with what arrived # before then: cut short, not lost. Its read carries on in the background # until that program lets go of it (it's killed with PowerShell at the - # latest). A stream whose read failed (StreamError) comes back $null. + # latest), dropping what arrives after: the text handed over is that + # stream's text from then on, and calling this again on the same job + # returns the same. A stream whose read failed comes back $null, with the + # reason in StreamError, even if the other stream timed out. # .PARAMETER Job # What Start-PkProcess returned, or a dispatcher job with the same keys. # .PARAMETER StreamTimeoutMs @@ -1793,12 +1826,20 @@ function Complete-PkProcess { } catch { $result.StreamError = [string]$_.Exception.Message } + # The failed read's own reason, rather than WaitAll's wrapper. Read + # from the tasks: WaitAll throws only once every task has finished, + # so it says nothing of a read that failed while the other stream was + # still held open. + foreach ($task in $tasks) { + if ($task.IsFaulted) { $result.StreamError = [string]$task.Exception.GetBaseException().Message; break } + } } # The output's end (both streams closed, or the timeout) is separate from # the program's end (the exit code, below): a program it started can hold # a stream open after it exits. A stream still open at the timeout gives # the text that arrived before it, each stream on its own; its read - # carries on in the background until that program lets go. + # carries on in the background until that program lets go, dropping + # what arrives after (Snapshot keeps the text it returns). $ranToCompletion = [System.Threading.Tasks.TaskStatus]::RanToCompletion foreach ($stream in 'Stdout', 'Stderr') { $task = $Job[$stream + 'Task'] diff --git a/README.md b/README.md index a867422..0e26743 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth - `Aborted` is `$true` for a command an abort killed or kept from starting. - `TimedOut` is `$true` for a program `-TimeoutSeconds` killed. - `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`. -- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it. `StreamError` means a read failed, and that stream comes back `$null`. +- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open. `Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`.