From 864715c20e8427ba9c8f5e3aeaaaa09bc0896548 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:28:15 -0500 Subject: [PATCH 1/2] Check mandatory worker parameters up front; take a string of digits as a code page - Invoke-PkRunspaceJobs refuses a job that leaves out a parameter the worker makes [Parameter(Mandatory)]: before anything starts for a fixed -JobParams list, as it arrives for -NextJobProvider. The job used to fail in binding and land in Errors while the rest ran. CancelToken is left out of the check (it's filled in), and so are workers with parameter sets, where what's needed depends on the set. - -OutputEncoding takes '65001' as a string as a code page, as from a CSV or config file, instead of refusing it as an unknown name. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 37 ++++++++++++++++++++++++++++ ParallelKit.ps1 | 53 ++++++++++++++++++++++++++++++++++++----- README.md | 4 ++-- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 014897e..b1c68d3 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4683,6 +4683,43 @@ try { } Write-Host "" +Write-Host "=== BC: up-front checks: mandatory worker parameters, a code page as a string ===" -ForegroundColor Cyan +Reset-Harness +# A job that leaves out a mandatory parameter is refused before anything +# starts, like a key the worker doesn't declare. It used to run, fail in +# binding, and land in Errors. +$bcRan = [System.Collections.Generic.List[object]]::new() +$bcError = $null +try { + $null = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Name = 'a'; Id = 1 }, @{ Id = 2 }) -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Id) $Id } -ResultHandler { param($r) $bcRan.Add($r) }.GetNewClosure() +} catch { $bcError = [string]$_ } +Test-Assert ($bcError -match "job 1 doesn't supply 'Name'" -and $bcError -match 'Nothing was started' -and $bcRan.Count -eq 0) ("-JobParams: a job missing a mandatory parameter is refused before anything starts (" + $bcError + "; ran " + $bcRan.Count + ")") +# The same for a -NextJobProvider job, as it arrives. +$bcFeed = [System.Collections.Generic.Queue[hashtable]]::new(); $bcFeed.Enqueue(@{ Name = 'a' }); $bcFeed.Enqueue(@{ Other = 1 }) +$bcError = $null +try { + $null = Invoke-PkRunspaceJobs -MaxParallel 1 -NextJobProvider { if ($bcFeed.Count -gt 0) { return $bcFeed.Dequeue() }; return $null }.GetNewClosure() -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Other) $Name } +} catch { $bcError = [string]$_ } +Test-Assert ($bcError -match "doesn't supply 'Name'") ("-NextJobProvider: a job missing a mandatory parameter is refused as it arrives (" + $bcError + ")") +# Not refused: a mandatory CancelToken (the kit supplies it), Mandatory = +# $false, and parameter sets (which parameters are needed depends on the set). +$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Id = 1 }) -WorkerScript { param([Parameter(Mandatory)][System.Threading.CancellationToken]$CancelToken, [Parameter(Mandatory = $false)]$Opt, $Id) $Id } +Test-Assert ($bcOut.Errors.Count -eq 0) ("a mandatory CancelToken and Mandatory = `$false aren't counted as missing (" + $bcOut.Errors.Count + " errors)") +$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ A = 1 }) -WorkerScript { param([Parameter(Mandatory, ParameterSetName = 'A')]$A, [Parameter(Mandatory, ParameterSetName = 'B')]$B) $A } +Test-Assert ($bcOut.Errors.Count -eq 0) ("a worker with parameter sets isn't refused for the other set's mandatory parameter (" + $bcOut.Errors.Count + " errors)") + +# A code page read from a CSV or config file is a string. +$bcResolve = { param($e) & (Get-Module ParallelKit) { param($x) try { (Resolve-PkEncoding -Encoding $x).CodePage } catch { 'refused' } } $e } +$bcCodePages = @((& $bcResolve '65001'), (& $bcResolve ' 437 '), (& $bcResolve 65001)) +Test-Assert ($bcCodePages[0] -eq 65001 -and $bcCodePages[1] -eq 437 -and $bcCodePages[2] -eq 65001) ("-OutputEncoding '65001' as a string is taken as code page 65001, like the number (" + ($bcCodePages -join ', ') + ")") +$bcError = $null +try { $null = Invoke-PkProcessBatch -OutputEncoding '99999999999' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $bcError = [string]$_ } +Test-Assert ($bcError -like "*'99999999999' isn't an encoding*") ("a string of digits that isn't a code page is still refused (" + $bcError + ")") +$bcOk = $null +try { $bcOk = (Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/d /c echo ok' -OutputEncoding '65001').Stdout } catch { $bcOk = [string]$_ } +Test-Assert ($bcOk -match '^ok') ("a program runs with -OutputEncoding '65001' ('" + ([string]$bcOk).Trim() + "')") +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index bd3fc15..86d7418 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1337,13 +1337,16 @@ function Resolve-PkArguments { function Resolve-PkEncoding { # -OutputEncoding as given (an Encoding, a code page number, or a name - # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". + # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". A code + # page can come as a string of digits too ('65001' from a CSV or config). param($Encoding) if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null } if ($Encoding -is [System.Text.Encoding]) { return $Encoding } # .NET Framework doesn't know the name 'utf8'; no BOM either way, since # this only decodes. if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) } + $codePage = 0 + if ($Encoding -is [string] -and $Encoding.Trim() -match '^[0-9]+$' -and [int]::TryParse($Encoding.Trim(), [ref]$codePage)) { $Encoding = $codePage } try { if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) } return [System.Text.Encoding]::GetEncoding([string]$Encoding) @@ -1417,8 +1420,8 @@ function Start-PkProcess { # paths, so a relative path is relative to your current location, and drives # like Data:\ or ~ work. # .PARAMETER OutputEncoding - # How to decode its output: a name like 'utf-8', a code page number, or a - # [System.Text.Encoding]. Default: the console's code page. + # How to decode its output: a name like 'utf-8', a code page number (65001 + # or '65001'), or a [System.Text.Encoding]. Default: the console's code page. # .OUTPUTS # A hashtable @{ Proc; StdoutTask; StderrTask; Stopwatch }. Proc is the # program's Process. Read its output from StdoutTask and StderrTask: Proc's @@ -2707,8 +2710,8 @@ function Invoke-PkProcessBatch { # Kill a program still running after this long (default 0, no limit). Its # result has TimedOut = $true. # .PARAMETER OutputEncoding - # How to decode every program's output: a name like 'utf-8', a code page number, - # or a [System.Text.Encoding]. Default: the console's code page. + # How to decode every program's output: a name like 'utf-8', a code page number + # (65001 or '65001'), or a [System.Text.Encoding]. Default: the console's code page. # .OUTPUTS # An object with Results, Aborted and ForceKilledCount. Each result has Command, # ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, StartError, Aborted, @@ -3055,7 +3058,8 @@ function Invoke-PkRunspaceJobs { # $CancelToken, it gets the run's cancellation token. # .PARAMETER JobParams # A list or array of job hashtables, or a single one. Use this or - # -NextJobProvider. + # -NextJobProvider. Checked before anything starts: a key the worker + # doesn't declare, or a mandatory parameter a job leaves out, is an error. # .PARAMETER NextJobProvider # { } returning the next job hashtable, or $null when there are no more. Runs on # the main thread, so jobs can be produced lazily. Only its last value counts: @@ -3162,6 +3166,33 @@ function Invoke-PkRunspaceJobs { if ($null -ne $WorkerScript.Ast.ParamBlock) { foreach ($p in $WorkerScript.Ast.ParamBlock.Parameters) { $null = $declaredParams.Add($p.Name.VariablePath.UserPath) } } + # Leaving out a mandatory one fails that job in binding, after others have + # started. Not checked with parameter sets, where what's needed depends on + # the set, and not for CancelToken, which is filled in when a job leaves + # it out. + $mandatoryParams = [System.Collections.Generic.List[string]]::new() + $hasParameterSets = $false + if ($null -ne $WorkerScript.Ast.ParamBlock) { + foreach ($workerParam in $WorkerScript.Ast.ParamBlock.Parameters) { + $isMandatory = $false + foreach ($paramAttr in $workerParam.Attributes) { + if ($paramAttr -isnot [System.Management.Automation.Language.AttributeAst]) { continue } + if ($paramAttr.TypeName.GetReflectionAttributeType() -ne [System.Management.Automation.ParameterAttribute]) { continue } + foreach ($namedArg in $paramAttr.NamedArguments) { + if ($namedArg.ArgumentName -eq 'ParameterSetName') { $hasParameterSets = $true } + if ($namedArg.ArgumentName -ne 'Mandatory') { continue } + $mandatoryValue = $true + if (-not $namedArg.ExpressionOmitted) { + try { $mandatoryValue = [bool]$namedArg.Argument.SafeGetValue() } catch { $mandatoryValue = $false } # not a constant: left to binding, as before + } + if ($mandatoryValue) { $isMandatory = $true } + } + } + $workerParamName = $workerParam.Name.VariablePath.UserPath + if ($isMandatory -and $workerParamName -ne 'CancelToken') { $mandatoryParams.Add($workerParamName) } + } + } + if ($hasParameterSets) { $mandatoryParams.Clear() } # A fixed list can be checked before a single worker starts. if ($null -ne $JobParams) { @@ -3171,6 +3202,11 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.") } } + foreach ($mandatoryParam in $mandatoryParams) { + if (-not $JobParams[$jobIndex].ContainsKey($mandatoryParam)) { + throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory. Nothing was started.") + } + } if ($checkSafety) { $sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex] if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") } @@ -3281,6 +3317,11 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.") } } + foreach ($mandatoryParam in $mandatoryParams) { + if (-not $nextJob.ContainsKey($mandatoryParam)) { + throw ("Invoke-PkRunspaceJobs: a job from -NextJobProvider doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory.") + } + } if ($checkSafety -and $null -eq $JobParams) { $sbKey = Get-PkScriptBlockJobKey -Job $nextJob if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) } diff --git a/README.md b/README.md index f019c36..bd9aab8 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`. -Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). +Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`, or the same as a string, as read from a CSV or config file) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. @@ -186,7 +186,7 @@ $out = Invoke-PkRun { Invoke-PkRunspaceJobs @jobsParams } Each worker runs in its own runspace: a separate PowerShell session inside the same process, with its own variables and functions. That separation is what makes running them at the same time safe, and also why a worker can't see your script's variables or functions unless you hand them over. -- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key. A key it doesn't declare is an error; with a fixed `-JobParams` list it's caught before any worker starts. +- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key, and every job must supply each parameter it makes `[Parameter(Mandatory)]` (`CancelToken` aside, which is filled in). Either mistake is an error; with a fixed `-JobParams` list it's caught before any worker starts, and a `-NextJobProvider` job is checked as it arrives. A worker with parameter sets isn't checked for mandatory ones, since what it needs depends on the set. - If the worker declares a `$CancelToken` parameter, it gets the run's cancellation token. - `-JobParams` takes a list or array of job hashtables. A single hashtable works too, so a list that happens to hold one job is fine. - A worker runspace starts empty. Name every one of your functions it calls in `-SharedFunctionNames`. The kit's own worker functions are always there, and cmdlets and functions from installed modules load on their own. A module you imported by path isn't installed, so list its functions too (or put it on `$env:PSModulePath`). From ba603d0b4fc75d8733658bfd68324174aa1b97d1 Mon Sep 17 00:00:00 2001 From: Hal <22532613+HalRiveria@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:29:29 -0500 Subject: [PATCH 2/2] Mandatory check ignores key case; any whole number is a code page - The mandatory-parameter check compares names case-insensitively, as binding does, so a case-sensitive [hashtable]::new() job isn't refused. One pass over the param block now collects both sets. - -OutputEncoding takes any whole number as a code page, such as the Int64 ConvertFrom-Json gives on PowerShell 7. Co-Authored-By: Claude Opus 5.5 --- ParallelKit.Harness.ps1 | 8 +++++++- ParallelKit.ps1 | 14 ++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index b1c68d3..61ea44e 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4701,6 +4701,11 @@ try { $null = Invoke-PkRunspaceJobs -MaxParallel 1 -NextJobProvider { if ($bcFeed.Count -gt 0) { return $bcFeed.Dequeue() }; return $null }.GetNewClosure() -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Other) $Name } } catch { $bcError = [string]$_ } Test-Assert ($bcError -match "doesn't supply 'Name'") ("-NextJobProvider: a job missing a mandatory parameter is refused as it arrives (" + $bcError + ")") +# A case-sensitive hashtable's 'name' binds to $Name, so it isn't missing. +$bcJob = [hashtable]::new(); $bcJob['name'] = 'a' +$bcOut = $null; $bcError = $null +try { $bcOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(,$bcJob) -WorkerScript { param([Parameter(Mandatory)][string]$Name) $Name } } catch { $bcError = [string]$_ } +Test-Assert ($null -eq $bcError -and $null -ne $bcOut -and $bcOut.Errors.Count -eq 0) ("a mandatory parameter supplied in another case, in a case-sensitive hashtable, isn't counted as missing (" + $bcError + ")") # Not refused: a mandatory CancelToken (the kit supplies it), Mandatory = # $false, and parameter sets (which parameters are needed depends on the set). $bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Id = 1 }) -WorkerScript { param([Parameter(Mandatory)][System.Threading.CancellationToken]$CancelToken, [Parameter(Mandatory = $false)]$Opt, $Id) $Id } @@ -4710,8 +4715,9 @@ Test-Assert ($bcOut.Errors.Count -eq 0) ("a worker with parameter sets isn't ref # A code page read from a CSV or config file is a string. $bcResolve = { param($e) & (Get-Module ParallelKit) { param($x) try { (Resolve-PkEncoding -Encoding $x).CodePage } catch { 'refused' } } $e } -$bcCodePages = @((& $bcResolve '65001'), (& $bcResolve ' 437 '), (& $bcResolve 65001)) +$bcCodePages = @((& $bcResolve '65001'), (& $bcResolve ' 437 '), (& $bcResolve 65001), (& $bcResolve ([long]65001))) Test-Assert ($bcCodePages[0] -eq 65001 -and $bcCodePages[1] -eq 437 -and $bcCodePages[2] -eq 65001) ("-OutputEncoding '65001' as a string is taken as code page 65001, like the number (" + ($bcCodePages -join ', ') + ")") +Test-Assert ($bcCodePages[3] -eq 65001) ("so is 65001 as an Int64, as ConvertFrom-Json gives it on PowerShell 7 (" + $bcCodePages[3] + ")") $bcError = $null try { $null = Invoke-PkProcessBatch -OutputEncoding '99999999999' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $bcError = [string]$_ } Test-Assert ($bcError -like "*'99999999999' isn't an encoding*") ("a string of digits that isn't a code page is still refused (" + $bcError + ")") diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index 86d7418..003ee1d 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1345,8 +1345,10 @@ function Resolve-PkEncoding { # .NET Framework doesn't know the name 'utf8'; no BOM either way, since # this only decodes. if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) } + # Any whole number, too: ConvertFrom-Json on 7 gives an Int64. $codePage = 0 - if ($Encoding -is [string] -and $Encoding.Trim() -match '^[0-9]+$' -and [int]::TryParse($Encoding.Trim(), [ref]$codePage)) { $Encoding = $codePage } + $digits = ([string]$Encoding).Trim() + if ($digits -match '^[0-9]+$' -and [int]::TryParse($digits, [ref]$codePage)) { $Encoding = $codePage } try { if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) } return [System.Text.Encoding]::GetEncoding([string]$Encoding) @@ -3163,9 +3165,6 @@ function Invoke-PkRunspaceJobs { # Passing a parameter the worker doesn't declare fails the whole invoke # with an unhelpful binding error. $declaredParams = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) - if ($null -ne $WorkerScript.Ast.ParamBlock) { - foreach ($p in $WorkerScript.Ast.ParamBlock.Parameters) { $null = $declaredParams.Add($p.Name.VariablePath.UserPath) } - } # Leaving out a mandatory one fails that job in binding, after others have # started. Not checked with parameter sets, where what's needed depends on # the set, and not for CancelToken, which is filled in when a job leaves @@ -3174,6 +3173,7 @@ function Invoke-PkRunspaceJobs { $hasParameterSets = $false if ($null -ne $WorkerScript.Ast.ParamBlock) { foreach ($workerParam in $WorkerScript.Ast.ParamBlock.Parameters) { + $null = $declaredParams.Add($workerParam.Name.VariablePath.UserPath) $isMandatory = $false foreach ($paramAttr in $workerParam.Attributes) { if ($paramAttr -isnot [System.Management.Automation.Language.AttributeAst]) { continue } @@ -3202,8 +3202,10 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.") } } + # -contains, not ContainsKey: case-insensitive like binding, even + # for a case-sensitive [hashtable]::new() job. foreach ($mandatoryParam in $mandatoryParams) { - if (-not $JobParams[$jobIndex].ContainsKey($mandatoryParam)) { + if (@($JobParams[$jobIndex].Keys) -notcontains $mandatoryParam) { throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory. Nothing was started.") } } @@ -3318,7 +3320,7 @@ function Invoke-PkRunspaceJobs { } } foreach ($mandatoryParam in $mandatoryParams) { - if (-not $nextJob.ContainsKey($mandatoryParam)) { + if (@($nextJob.Keys) -notcontains $mandatoryParam) { # case-insensitive, as for -JobParams above throw ("Invoke-PkRunspaceJobs: a job from -NextJobProvider doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory.") } }