Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4546,6 +4546,71 @@ try {
}
Write-Host ""

Write-Host "=== BA: after a stream timeout, the read drains without keeping the text ===" -ForegroundColor Cyan
# The read of a held stream goes on to its end, but nobody can get what it
# reads after the result was handed over, so it isn't kept. The reader is
# closed at the end of every stream, rather than at the next garbage
# collection. Test-only reads that a build without them lacks count as a
# failure, not an error.
Reset-Harness
function Get-HarnessReadState($Task, [string]$Name) {
try { return $Task.$Name } catch { return 'missing' } # an older build has no such test-only read
}
function Stop-HarnessPingsOf([int]$ParentId) {
foreach ($p in @(Get-CimInstance Win32_Process -Filter ("ParentProcessId=" + $ParentId + " AND Name='PING.EXE'"))) { Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue }
}
# A ping that writes a line a second, holding the output after cmd exits.
$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo first& start /b ping -t 127.0.0.1'
try {
$null = $s['Proc'].WaitForExit(10000)
$r = Complete-PkProcess $s -StreamTimeoutMs 1500
$baHeldAtResult = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength'
Start-Sleep -Seconds 3
$baHeldLater = Get-HarnessReadState $s['StdoutTask'] 'TestHeldLength'
$baStillReading = -not $s['StdoutTask'].IsCompleted
$baAgain = Complete-PkProcess $s -StreamTimeoutMs 0
$baLength = ([string]$r.Stdout).Length
Test-Assert ($baAgain.Stdout -ceq $r.Stdout) ("a second Complete-PkProcess on the same job returns the same text (" + ([string]$baAgain.Stdout).Length + " characters, first " + $baLength + ")")
Test-Assert ($r.StreamTimedOut -and $r.Stdout -match 'first' -and $baStillReading -and $baHeldAtResult -eq $baLength -and $baHeldLater -eq $baLength) ("a program that keeps writing after the result: the buffer stays at the result's " + $baLength + " characters (at the result " + $baHeldAtResult + ", 3 s later " + $baHeldLater + ", still reading " + $baStillReading + ")")
} finally { Stop-HarnessPingsOf -ParentId $s['Proc'].Id }
$baEnded = $s['StdoutTask'].Wait(10000) -and $s['StderrTask'].Wait(10000)
$baFinal = $null; if ($baEnded) { $baFinal = $s['StdoutTask'].Result }
Test-Assert ($baEnded -and $baFinal -eq $r.Stdout -and ([string]$baFinal).Length -eq $baLength) ("once the writer is killed the read ends, and its result is the text handed over, not what came after (" + $baEnded + ", " + ([string]$baFinal).Length + " characters)")
Test-Assert ((Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed') -eq $true) "a held stream's reader is closed when the stream ends"
$s['Proc'].Dispose()

foreach ($baPath in 'created in the job', 'Process.Start fallback') {
[PkChildProcessJob]::TestUseProcessStart = ($baPath -eq 'Process.Start fallback')
try {
$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c echo out& echo err 1>&2'
$r = Complete-PkProcess $s -Dispose
} finally { [PkChildProcessJob]::TestUseProcessStart = $false }
$baOut = Get-HarnessReadState $s['StdoutTask'] 'TestReaderClosed'
$baErr = Get-HarnessReadState $s['StderrTask'] 'TestReaderClosed'
Test-Assert ($r.Stdout -match 'out' -and $r.Stderr -match 'err' -and $baOut -eq $true -and $baErr -eq $true) ($baPath + ": both readers are closed once a program's output ends (stdout " + $baOut + ", stderr " + $baErr + ")")
}

# One read fails while the other stream is still held: the failure is
# reported, not just the timeout. Task.WaitAll with a timeout throws only once
# every task has finished, so it didn't throw here.
$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0'
$null = $s['Proc'].WaitForExit(10000)
$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000)
$baFailed = [System.Threading.Tasks.TaskCompletionSource[string]]::new()
$baFailed.SetException([System.IO.IOException]::new('Test: the read failed.'))
$baHeld = [System.Threading.Tasks.TaskCompletionSource[string]]::new()
$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $baHeld.Task } -StreamTimeoutMs 300 -Dispose
$baHeld.SetResult('')
Test-Assert ($r.StreamTimedOut -and $r.StreamError -match 'the read failed' -and $null -eq $r.Stdout) ("a read that failed while the other stream timed out is reported in StreamError (StreamTimedOut " + $r.StreamTimedOut + ", StreamError '" + $r.StreamError + "')")
# And when the other stream ended normally: the read's own reason, not
# WaitAll's "One or more errors occurred".
$s = Start-PkProcess -FilePath $cmdExe -Arguments '/d /c exit 0'
$null = $s['Proc'].WaitForExit(10000)
$null = [System.Threading.Tasks.Task]::WaitAll(@($s['StdoutTask'], $s['StderrTask']), 10000)
$r = Complete-PkProcess @{ Proc = $s['Proc']; StdoutTask = $baFailed.Task; StderrTask = $s['StderrTask'] } -StreamTimeoutMs 300 -Dispose
Test-Assert (-not $r.StreamTimedOut -and $r.StreamError -eq 'Test: the read failed.' -and $null -eq $r.Stdout) ("a failed read next to one that ended gives the read's own reason (StreamError '" + $r.StreamError + "')")
Write-Host ""

Set-PkMessageHandler -Handler $null
Reset-PkAbortState

Expand Down
73 changes: 57 additions & 16 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ public static class PkState

// Bump this, and the check after the type definitions, whenever any C#
// in this file changes: a session keeps the types it compiled first.
public const int TypesVersion = 8;
public const int TypesVersion = 9;

// Locked: Register-/Unregister-PkCancellationSource are callable from
// worker runspaces, and List<T> is not safe for concurrent writers.
Expand Down Expand Up @@ -247,8 +247,12 @@ public sealed class PkStartedProcess

// Reads one of a program's output streams to its end, in chunks, so what has
// arrived can be taken at any time (Snapshot): a program the program started
// can hold the stream open long after it exits. The task's result is the
// whole text, at the end of the stream. The reader's decoder carries a
// can hold the stream open long after it exits. The first snapshot taken
// before the end is the stream's text from then on: the read carries on to
// the end, so the program holding the stream never blocks on a full pipe,
// but drops what it reads, which nobody can get any more. The task's result
// is that snapshot, or the whole text if none was taken. The reader is
// closed at the end of the stream. The reader's decoder carries a
// character split across two reads over to the next, so a snapshot never
// ends in half a character. Each read runs on a thread of its own
// (LongRunning), not a thread-pool thread: a read blocks until every holder
Expand All @@ -261,23 +265,43 @@ public sealed class PkStreamReadTask : Task<string>
private sealed class Buffer
{
private readonly TextReader _reader;
// Null once the text is kept: at a snapshot or at the end of the
// stream, whichever comes first. The builder goes then, so the text
// isn't held twice.
private StringBuilder _text = new StringBuilder();
private string _final;
private string _kept;
private readonly object _gate = new object();
public volatile bool ReaderClosed;
public Buffer(TextReader reader) { _reader = reader; }
public string Pump()
{
char[] chunk = new char[4096];
int n;
while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0)
try
{
char[] chunk = new char[4096];
int n;
while ((n = _reader.Read(chunk, 0, chunk.Length)) > 0)
{
lock (_gate) { if (_text != null) { _text.Append(chunk, 0, n); } }
}
return Keep();
}
finally
{
lock (_gate) { _text.Append(chunk, 0, n); }
// Closes the pipe now rather than at the next garbage collection.
try { _reader.Dispose(); ReaderClosed = true; }
catch (Exception) { } // a close that fails must not hide the read's own result or error
}
// The builder goes once the text is final, so it isn't held twice.
lock (_gate) { _final = _text.ToString(); _text = null; return _final; }
}
// Locked: the reader may be appending while Complete-PkProcess takes it.
public string Snapshot() { lock (_gate) { return _final != null ? _final : _text.ToString(); } }
public string Keep()
{
lock (_gate)
{
if (_text != null) { _kept = _text.ToString(); _text = null; }
return _kept;
}
}
public int HeldLength { get { lock (_gate) { return _text != null ? _text.Length : _kept.Length; } } }
}

private readonly Buffer _buffer;
Expand All @@ -290,8 +314,14 @@ public sealed class PkStreamReadTask : Task<string>
return task;
}

// The text that has arrived so far; all of it once the task has finished.
public string Snapshot() { return _buffer.Snapshot(); }
// The text that has arrived so far, kept as the stream's text from then
// on; all of it if the stream has already ended.
public string Snapshot() { return _buffer.Keep(); }

// Test-only reads: how many characters the buffer holds, and whether the
// reader has been closed.
public int TestHeldLength { get { return _buffer.HeldLength; } }
public bool TestReaderClosed { get { return _buffer.ReaderClosed; } }
}

// A Windows Job Object with KILL_ON_JOB_CLOSE: every child process assigned
Expand Down Expand Up @@ -782,7 +812,7 @@ public static class PkChildProcessJob
# A session that loaded an older copy keeps its compiled types. Say so,
# rather than fail later with a missing-method error. Keep this number in
# sync with [PkState]::TypesVersion.
if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 8) {
if ((& { try { [PkState]::TypesVersion } catch { 0 } }) -ne 9) {
throw "ParallelKit: this PowerShell session already loaded an older version of the kit, and its compiled parts can't be swapped out. Close this window and start a new one."
}

Expand Down Expand Up @@ -1753,7 +1783,10 @@ function Complete-PkProcess {
# that limit was hit. The stream it gave up on comes back with what arrived
# before then: cut short, not lost. Its read carries on in the background
# until that program lets go of it (it's killed with PowerShell at the
# latest). A stream whose read failed (StreamError) comes back $null.
# latest), dropping what arrives after: the text handed over is that
# stream's text from then on, and calling this again on the same job
# returns the same. A stream whose read failed comes back $null, with the
# reason in StreamError, even if the other stream timed out.
# .PARAMETER Job
# What Start-PkProcess returned, or a dispatcher job with the same keys.
# .PARAMETER StreamTimeoutMs
Expand Down Expand Up @@ -1793,12 +1826,20 @@ function Complete-PkProcess {
} catch {
$result.StreamError = [string]$_.Exception.Message
}
# The failed read's own reason, rather than WaitAll's wrapper. Read
# from the tasks: WaitAll throws only once every task has finished,
# so it says nothing of a read that failed while the other stream was
# still held open.
foreach ($task in $tasks) {
if ($task.IsFaulted) { $result.StreamError = [string]$task.Exception.GetBaseException().Message; break }
}
}
# The output's end (both streams closed, or the timeout) is separate from
# the program's end (the exit code, below): a program it started can hold
# a stream open after it exits. A stream still open at the timeout gives
# the text that arrived before it, each stream on its own; its read
# carries on in the background until that program lets go.
# carries on in the background until that program lets go, dropping
# what arrives after (Snapshot keeps the text it returns).
$ranToCompletion = [System.Threading.Tasks.TaskStatus]::RanToCompletion
foreach ($stream in 'Stdout', 'Stderr') {
$task = $Job[$stream + 'Task']
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth
- `Aborted` is `$true` for a command an abort killed or kept from starting.
- `TimedOut` is `$true` for a program `-TimeoutSeconds` killed.
- `Duration` is how long the program ran, from its start to its exit or kill, as a `[TimeSpan]`: not the time it waited for a slot, nor any wait for its output afterwards. It's `$null` for a command that never started. Sorting on it finds the slow ones: `$batch.Results | Sort-Object Duration -Descending | Select-Object -First 10`.
- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it. `StreamError` means a read failed, and that stream comes back `$null`.
- `StreamTimedOut` means the output read gave up 15 seconds after the program exited, on stdout, stderr or both, which only happens when a program it started is still holding the output open. Each stream comes back with what arrived before then: one held open is cut short, the other is whole. The read of a held stream carries on in the background until that program lets go of it; it's in the kit's Job Object, so that's when PowerShell exits at the latest. Until then that read keeps a thread of its own, one per held stream, so other programs' output isn't held up behind it, and drops what it reads, so a program that keeps writing doesn't fill memory with output nobody can get. What came back is that stream's text from then on, so calling `Complete-PkProcess` again on the same job returns the same text, and so does the read task's `Result` once the stream ends. `StreamError` means a read failed, and that stream comes back `$null`; it can be set alongside `StreamTimedOut` when the other stream was held open.

`Results` is in the order commands finished, not the order you gave them. Use `Command` to match them up, or give each command your own key (`Index = $n`) and sort on it: `$batch.Results | Sort-Object { $_.Command.Index }`.

Expand Down
Loading