Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions ParallelKit.Harness.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4683,6 +4683,49 @@ try {
}
Write-Host ""

Write-Host "=== BC: up-front checks: mandatory worker parameters, a code page as a string ===" -ForegroundColor Cyan
Reset-Harness
# A job that leaves out a mandatory parameter is refused before anything
# starts, like a key the worker doesn't declare. It used to run, fail in
# binding, and land in Errors.
$bcRan = [System.Collections.Generic.List[object]]::new()
$bcError = $null
try {
$null = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Name = 'a'; Id = 1 }, @{ Id = 2 }) -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Id) $Id } -ResultHandler { param($r) $bcRan.Add($r) }.GetNewClosure()
} catch { $bcError = [string]$_ }
Test-Assert ($bcError -match "job 1 doesn't supply 'Name'" -and $bcError -match 'Nothing was started' -and $bcRan.Count -eq 0) ("-JobParams: a job missing a mandatory parameter is refused before anything starts (" + $bcError + "; ran " + $bcRan.Count + ")")
# The same for a -NextJobProvider job, as it arrives.
$bcFeed = [System.Collections.Generic.Queue[hashtable]]::new(); $bcFeed.Enqueue(@{ Name = 'a' }); $bcFeed.Enqueue(@{ Other = 1 })
$bcError = $null
try {
$null = Invoke-PkRunspaceJobs -MaxParallel 1 -NextJobProvider { if ($bcFeed.Count -gt 0) { return $bcFeed.Dequeue() }; return $null }.GetNewClosure() -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Other) $Name }
} catch { $bcError = [string]$_ }
Test-Assert ($bcError -match "doesn't supply 'Name'") ("-NextJobProvider: a job missing a mandatory parameter is refused as it arrives (" + $bcError + ")")
# A case-sensitive hashtable's 'name' binds to $Name, so it isn't missing.
$bcJob = [hashtable]::new(); $bcJob['name'] = 'a'
$bcOut = $null; $bcError = $null
try { $bcOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(,$bcJob) -WorkerScript { param([Parameter(Mandatory)][string]$Name) $Name } } catch { $bcError = [string]$_ }
Test-Assert ($null -eq $bcError -and $null -ne $bcOut -and $bcOut.Errors.Count -eq 0) ("a mandatory parameter supplied in another case, in a case-sensitive hashtable, isn't counted as missing (" + $bcError + ")")
# Not refused: a mandatory CancelToken (the kit supplies it), Mandatory =
# $false, and parameter sets (which parameters are needed depends on the set).
$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Id = 1 }) -WorkerScript { param([Parameter(Mandatory)][System.Threading.CancellationToken]$CancelToken, [Parameter(Mandatory = $false)]$Opt, $Id) $Id }
Test-Assert ($bcOut.Errors.Count -eq 0) ("a mandatory CancelToken and Mandatory = `$false aren't counted as missing (" + $bcOut.Errors.Count + " errors)")
$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ A = 1 }) -WorkerScript { param([Parameter(Mandatory, ParameterSetName = 'A')]$A, [Parameter(Mandatory, ParameterSetName = 'B')]$B) $A }
Test-Assert ($bcOut.Errors.Count -eq 0) ("a worker with parameter sets isn't refused for the other set's mandatory parameter (" + $bcOut.Errors.Count + " errors)")

# A code page read from a CSV or config file is a string.
$bcResolve = { param($e) & (Get-Module ParallelKit) { param($x) try { (Resolve-PkEncoding -Encoding $x).CodePage } catch { 'refused' } } $e }
$bcCodePages = @((& $bcResolve '65001'), (& $bcResolve ' 437 '), (& $bcResolve 65001), (& $bcResolve ([long]65001)))
Test-Assert ($bcCodePages[0] -eq 65001 -and $bcCodePages[1] -eq 437 -and $bcCodePages[2] -eq 65001) ("-OutputEncoding '65001' as a string is taken as code page 65001, like the number (" + ($bcCodePages -join ', ') + ")")
Test-Assert ($bcCodePages[3] -eq 65001) ("so is 65001 as an Int64, as ConvertFrom-Json gives it on PowerShell 7 (" + $bcCodePages[3] + ")")
$bcError = $null
try { $null = Invoke-PkProcessBatch -OutputEncoding '99999999999' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $bcError = [string]$_ }
Test-Assert ($bcError -like "*'99999999999' isn't an encoding*") ("a string of digits that isn't a code page is still refused (" + $bcError + ")")
$bcOk = $null
try { $bcOk = (Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/d /c echo ok' -OutputEncoding '65001').Stdout } catch { $bcOk = [string]$_ }
Test-Assert ($bcOk -match '^ok') ("a program runs with -OutputEncoding '65001' ('" + ([string]$bcOk).Trim() + "')")
Write-Host ""

Set-PkMessageHandler -Handler $null
Reset-PkAbortState

Expand Down
57 changes: 50 additions & 7 deletions ParallelKit.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -1337,13 +1337,18 @@ function Resolve-PkArguments {

function Resolve-PkEncoding {
# -OutputEncoding as given (an Encoding, a code page number, or a name
# like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set".
# like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". A code
# page can come as a string of digits too ('65001' from a CSV or config).
param($Encoding)
if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null }
if ($Encoding -is [System.Text.Encoding]) { return $Encoding }
# .NET Framework doesn't know the name 'utf8'; no BOM either way, since
# this only decodes.
if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) }
# Any whole number, too: ConvertFrom-Json on 7 gives an Int64.
$codePage = 0
$digits = ([string]$Encoding).Trim()
if ($digits -match '^[0-9]+$' -and [int]::TryParse($digits, [ref]$codePage)) { $Encoding = $codePage }
try {
if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) }
return [System.Text.Encoding]::GetEncoding([string]$Encoding)
Expand Down Expand Up @@ -1417,8 +1422,8 @@ function Start-PkProcess {
# paths, so a relative path is relative to your current location, and drives
# like Data:\ or ~ work.
# .PARAMETER OutputEncoding
# How to decode its output: a name like 'utf-8', a code page number, or a
# [System.Text.Encoding]. Default: the console's code page.
# How to decode its output: a name like 'utf-8', a code page number (65001
# or '65001'), or a [System.Text.Encoding]. Default: the console's code page.
# .OUTPUTS
# A hashtable @{ Proc; StdoutTask; StderrTask; Stopwatch }. Proc is the
# program's Process. Read its output from StdoutTask and StderrTask: Proc's
Expand Down Expand Up @@ -2707,8 +2712,8 @@ function Invoke-PkProcessBatch {
# Kill a program still running after this long (default 0, no limit). Its
# result has TimedOut = $true.
# .PARAMETER OutputEncoding
# How to decode every program's output: a name like 'utf-8', a code page number,
# or a [System.Text.Encoding]. Default: the console's code page.
# How to decode every program's output: a name like 'utf-8', a code page number
# (65001 or '65001'), or a [System.Text.Encoding]. Default: the console's code page.
# .OUTPUTS
# An object with Results, Aborted and ForceKilledCount. Each result has Command,
# ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, StartError, Aborted,
Expand Down Expand Up @@ -3055,7 +3060,8 @@ function Invoke-PkRunspaceJobs {
# $CancelToken, it gets the run's cancellation token.
# .PARAMETER JobParams
# A list or array of job hashtables, or a single one. Use this or
# -NextJobProvider.
# -NextJobProvider. Checked before anything starts: a key the worker
# doesn't declare, or a mandatory parameter a job leaves out, is an error.
# .PARAMETER NextJobProvider
# { } returning the next job hashtable, or $null when there are no more. Runs on
# the main thread, so jobs can be produced lazily. Only its last value counts:
Expand Down Expand Up @@ -3159,9 +3165,34 @@ function Invoke-PkRunspaceJobs {
# Passing a parameter the worker doesn't declare fails the whole invoke
# with an unhelpful binding error.
$declaredParams = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
# Leaving out a mandatory one fails that job in binding, after others have
# started. Not checked with parameter sets, where what's needed depends on
# the set, and not for CancelToken, which is filled in when a job leaves
# it out.
$mandatoryParams = [System.Collections.Generic.List[string]]::new()
$hasParameterSets = $false
if ($null -ne $WorkerScript.Ast.ParamBlock) {
foreach ($p in $WorkerScript.Ast.ParamBlock.Parameters) { $null = $declaredParams.Add($p.Name.VariablePath.UserPath) }
foreach ($workerParam in $WorkerScript.Ast.ParamBlock.Parameters) {
$null = $declaredParams.Add($workerParam.Name.VariablePath.UserPath)
$isMandatory = $false
foreach ($paramAttr in $workerParam.Attributes) {
if ($paramAttr -isnot [System.Management.Automation.Language.AttributeAst]) { continue }
if ($paramAttr.TypeName.GetReflectionAttributeType() -ne [System.Management.Automation.ParameterAttribute]) { continue }
foreach ($namedArg in $paramAttr.NamedArguments) {
if ($namedArg.ArgumentName -eq 'ParameterSetName') { $hasParameterSets = $true }
if ($namedArg.ArgumentName -ne 'Mandatory') { continue }
$mandatoryValue = $true
if (-not $namedArg.ExpressionOmitted) {
try { $mandatoryValue = [bool]$namedArg.Argument.SafeGetValue() } catch { $mandatoryValue = $false } # not a constant: left to binding, as before
}
if ($mandatoryValue) { $isMandatory = $true }
}
}
$workerParamName = $workerParam.Name.VariablePath.UserPath
if ($isMandatory -and $workerParamName -ne 'CancelToken') { $mandatoryParams.Add($workerParamName) }
}
}
if ($hasParameterSets) { $mandatoryParams.Clear() }

# A fixed list can be checked before a single worker starts.
if ($null -ne $JobParams) {
Expand All @@ -3171,6 +3202,13 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.")
}
}
# -contains, not ContainsKey: case-insensitive like binding, even
# for a case-sensitive [hashtable]::new() job.
foreach ($mandatoryParam in $mandatoryParams) {
if (@($JobParams[$jobIndex].Keys) -notcontains $mandatoryParam) {
throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory. Nothing was started.")
}
}
if ($checkSafety) {
$sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex]
if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") }
Expand Down Expand Up @@ -3281,6 +3319,11 @@ function Invoke-PkRunspaceJobs {
throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.")
}
}
foreach ($mandatoryParam in $mandatoryParams) {
if (@($nextJob.Keys) -notcontains $mandatoryParam) { # case-insensitive, as for -JobParams above
throw ("Invoke-PkRunspaceJobs: a job from -NextJobProvider doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory.")
}
}
if ($checkSafety -and $null -eq $JobParams) {
$sbKey = Get-PkScriptBlockJobKey -Job $nextJob
if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) }
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth

`-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`.

Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter).
Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`, or the same as a string, as read from a CSV or config file) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter).

`Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces.

Expand Down Expand Up @@ -186,7 +186,7 @@ $out = Invoke-PkRun { Invoke-PkRunspaceJobs @jobsParams }

Each worker runs in its own runspace: a separate PowerShell session inside the same process, with its own variables and functions. That separation is what makes running them at the same time safe, and also why a worker can't see your script's variables or functions unless you hand them over.

- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key. A key it doesn't declare is an error; with a fixed `-JobParams` list it's caught before any worker starts.
- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key, and every job must supply each parameter it makes `[Parameter(Mandatory)]` (`CancelToken` aside, which is filled in). Either mistake is an error; with a fixed `-JobParams` list it's caught before any worker starts, and a `-NextJobProvider` job is checked as it arrives. A worker with parameter sets isn't checked for mandatory ones, since what it needs depends on the set.
- If the worker declares a `$CancelToken` parameter, it gets the run's cancellation token.
- `-JobParams` takes a list or array of job hashtables. A single hashtable works too, so a list that happens to hold one job is fine.
- A worker runspace starts empty. Name every one of your functions it calls in `-SharedFunctionNames`. The kit's own worker functions are always there, and cmdlets and functions from installed modules load on their own. A module you imported by path isn't installed, so list its functions too (or put it on `$env:PSModulePath`).
Expand Down
Loading