diff --git a/ParallelKit.Harness.ps1 b/ParallelKit.Harness.ps1 index 014897e..61ea44e 100644 --- a/ParallelKit.Harness.ps1 +++ b/ParallelKit.Harness.ps1 @@ -4683,6 +4683,49 @@ try { } Write-Host "" +Write-Host "=== BC: up-front checks: mandatory worker parameters, a code page as a string ===" -ForegroundColor Cyan +Reset-Harness +# A job that leaves out a mandatory parameter is refused before anything +# starts, like a key the worker doesn't declare. It used to run, fail in +# binding, and land in Errors. +$bcRan = [System.Collections.Generic.List[object]]::new() +$bcError = $null +try { + $null = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Name = 'a'; Id = 1 }, @{ Id = 2 }) -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Id) $Id } -ResultHandler { param($r) $bcRan.Add($r) }.GetNewClosure() +} catch { $bcError = [string]$_ } +Test-Assert ($bcError -match "job 1 doesn't supply 'Name'" -and $bcError -match 'Nothing was started' -and $bcRan.Count -eq 0) ("-JobParams: a job missing a mandatory parameter is refused before anything starts (" + $bcError + "; ran " + $bcRan.Count + ")") +# The same for a -NextJobProvider job, as it arrives. +$bcFeed = [System.Collections.Generic.Queue[hashtable]]::new(); $bcFeed.Enqueue(@{ Name = 'a' }); $bcFeed.Enqueue(@{ Other = 1 }) +$bcError = $null +try { + $null = Invoke-PkRunspaceJobs -MaxParallel 1 -NextJobProvider { if ($bcFeed.Count -gt 0) { return $bcFeed.Dequeue() }; return $null }.GetNewClosure() -WorkerScript { param([Parameter(Mandatory)][string]$Name, $Other) $Name } +} catch { $bcError = [string]$_ } +Test-Assert ($bcError -match "doesn't supply 'Name'") ("-NextJobProvider: a job missing a mandatory parameter is refused as it arrives (" + $bcError + ")") +# A case-sensitive hashtable's 'name' binds to $Name, so it isn't missing. +$bcJob = [hashtable]::new(); $bcJob['name'] = 'a' +$bcOut = $null; $bcError = $null +try { $bcOut = Invoke-PkRunspaceJobs -MaxParallel 1 -JobParams @(,$bcJob) -WorkerScript { param([Parameter(Mandatory)][string]$Name) $Name } } catch { $bcError = [string]$_ } +Test-Assert ($null -eq $bcError -and $null -ne $bcOut -and $bcOut.Errors.Count -eq 0) ("a mandatory parameter supplied in another case, in a case-sensitive hashtable, isn't counted as missing (" + $bcError + ")") +# Not refused: a mandatory CancelToken (the kit supplies it), Mandatory = +# $false, and parameter sets (which parameters are needed depends on the set). +$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ Id = 1 }) -WorkerScript { param([Parameter(Mandatory)][System.Threading.CancellationToken]$CancelToken, [Parameter(Mandatory = $false)]$Opt, $Id) $Id } +Test-Assert ($bcOut.Errors.Count -eq 0) ("a mandatory CancelToken and Mandatory = `$false aren't counted as missing (" + $bcOut.Errors.Count + " errors)") +$bcOut = Invoke-PkRunspaceJobs -MaxParallel 2 -JobParams @(@{ A = 1 }) -WorkerScript { param([Parameter(Mandatory, ParameterSetName = 'A')]$A, [Parameter(Mandatory, ParameterSetName = 'B')]$B) $A } +Test-Assert ($bcOut.Errors.Count -eq 0) ("a worker with parameter sets isn't refused for the other set's mandatory parameter (" + $bcOut.Errors.Count + " errors)") + +# A code page read from a CSV or config file is a string. +$bcResolve = { param($e) & (Get-Module ParallelKit) { param($x) try { (Resolve-PkEncoding -Encoding $x).CodePage } catch { 'refused' } } $e } +$bcCodePages = @((& $bcResolve '65001'), (& $bcResolve ' 437 '), (& $bcResolve 65001), (& $bcResolve ([long]65001))) +Test-Assert ($bcCodePages[0] -eq 65001 -and $bcCodePages[1] -eq 437 -and $bcCodePages[2] -eq 65001) ("-OutputEncoding '65001' as a string is taken as code page 65001, like the number (" + ($bcCodePages -join ', ') + ")") +Test-Assert ($bcCodePages[3] -eq 65001) ("so is 65001 as an Int64, as ConvertFrom-Json gives it on PowerShell 7 (" + $bcCodePages[3] + ")") +$bcError = $null +try { $null = Invoke-PkProcessBatch -OutputEncoding '99999999999' -Commands @(@{ FilePath = $cmdExe; Arguments = '/c exit 0' }) } catch { $bcError = [string]$_ } +Test-Assert ($bcError -like "*'99999999999' isn't an encoding*") ("a string of digits that isn't a code page is still refused (" + $bcError + ")") +$bcOk = $null +try { $bcOk = (Invoke-PkProcessWithTimeout -FilePath $cmdExe -Arguments '/d /c echo ok' -OutputEncoding '65001').Stdout } catch { $bcOk = [string]$_ } +Test-Assert ($bcOk -match '^ok') ("a program runs with -OutputEncoding '65001' ('" + ([string]$bcOk).Trim() + "')") +Write-Host "" + Set-PkMessageHandler -Handler $null Reset-PkAbortState diff --git a/ParallelKit.ps1 b/ParallelKit.ps1 index bd3fc15..003ee1d 100644 --- a/ParallelKit.ps1 +++ b/ParallelKit.ps1 @@ -1337,13 +1337,18 @@ function Resolve-PkArguments { function Resolve-PkEncoding { # -OutputEncoding as given (an Encoding, a code page number, or a name - # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". + # like 'utf-8' or 'utf8') -> an Encoding, or $null for "not set". A code + # page can come as a string of digits too ('65001' from a CSV or config). param($Encoding) if ($null -eq $Encoding -or ($Encoding -is [string] -and $Encoding -eq '')) { return $null } if ($Encoding -is [System.Text.Encoding]) { return $Encoding } # .NET Framework doesn't know the name 'utf8'; no BOM either way, since # this only decodes. if ($Encoding -is [string] -and $Encoding -in @('utf8', 'utf-8')) { return [System.Text.UTF8Encoding]::new($false) } + # Any whole number, too: ConvertFrom-Json on 7 gives an Int64. + $codePage = 0 + $digits = ([string]$Encoding).Trim() + if ($digits -match '^[0-9]+$' -and [int]::TryParse($digits, [ref]$codePage)) { $Encoding = $codePage } try { if ($Encoding -is [int]) { return [System.Text.Encoding]::GetEncoding([int]$Encoding) } return [System.Text.Encoding]::GetEncoding([string]$Encoding) @@ -1417,8 +1422,8 @@ function Start-PkProcess { # paths, so a relative path is relative to your current location, and drives # like Data:\ or ~ work. # .PARAMETER OutputEncoding - # How to decode its output: a name like 'utf-8', a code page number, or a - # [System.Text.Encoding]. Default: the console's code page. + # How to decode its output: a name like 'utf-8', a code page number (65001 + # or '65001'), or a [System.Text.Encoding]. Default: the console's code page. # .OUTPUTS # A hashtable @{ Proc; StdoutTask; StderrTask; Stopwatch }. Proc is the # program's Process. Read its output from StdoutTask and StderrTask: Proc's @@ -2707,8 +2712,8 @@ function Invoke-PkProcessBatch { # Kill a program still running after this long (default 0, no limit). Its # result has TimedOut = $true. # .PARAMETER OutputEncoding - # How to decode every program's output: a name like 'utf-8', a code page number, - # or a [System.Text.Encoding]. Default: the console's code page. + # How to decode every program's output: a name like 'utf-8', a code page number + # (65001 or '65001'), or a [System.Text.Encoding]. Default: the console's code page. # .OUTPUTS # An object with Results, Aborted and ForceKilledCount. Each result has Command, # ExitCode, Stdout, Stderr, StreamTimedOut, StreamError, StartError, Aborted, @@ -3055,7 +3060,8 @@ function Invoke-PkRunspaceJobs { # $CancelToken, it gets the run's cancellation token. # .PARAMETER JobParams # A list or array of job hashtables, or a single one. Use this or - # -NextJobProvider. + # -NextJobProvider. Checked before anything starts: a key the worker + # doesn't declare, or a mandatory parameter a job leaves out, is an error. # .PARAMETER NextJobProvider # { } returning the next job hashtable, or $null when there are no more. Runs on # the main thread, so jobs can be produced lazily. Only its last value counts: @@ -3159,9 +3165,34 @@ function Invoke-PkRunspaceJobs { # Passing a parameter the worker doesn't declare fails the whole invoke # with an unhelpful binding error. $declaredParams = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + # Leaving out a mandatory one fails that job in binding, after others have + # started. Not checked with parameter sets, where what's needed depends on + # the set, and not for CancelToken, which is filled in when a job leaves + # it out. + $mandatoryParams = [System.Collections.Generic.List[string]]::new() + $hasParameterSets = $false if ($null -ne $WorkerScript.Ast.ParamBlock) { - foreach ($p in $WorkerScript.Ast.ParamBlock.Parameters) { $null = $declaredParams.Add($p.Name.VariablePath.UserPath) } + foreach ($workerParam in $WorkerScript.Ast.ParamBlock.Parameters) { + $null = $declaredParams.Add($workerParam.Name.VariablePath.UserPath) + $isMandatory = $false + foreach ($paramAttr in $workerParam.Attributes) { + if ($paramAttr -isnot [System.Management.Automation.Language.AttributeAst]) { continue } + if ($paramAttr.TypeName.GetReflectionAttributeType() -ne [System.Management.Automation.ParameterAttribute]) { continue } + foreach ($namedArg in $paramAttr.NamedArguments) { + if ($namedArg.ArgumentName -eq 'ParameterSetName') { $hasParameterSets = $true } + if ($namedArg.ArgumentName -ne 'Mandatory') { continue } + $mandatoryValue = $true + if (-not $namedArg.ExpressionOmitted) { + try { $mandatoryValue = [bool]$namedArg.Argument.SafeGetValue() } catch { $mandatoryValue = $false } # not a constant: left to binding, as before + } + if ($mandatoryValue) { $isMandatory = $true } + } + } + $workerParamName = $workerParam.Name.VariablePath.UserPath + if ($isMandatory -and $workerParamName -ne 'CancelToken') { $mandatoryParams.Add($workerParamName) } + } } + if ($hasParameterSets) { $mandatoryParams.Clear() } # A fixed list can be checked before a single worker starts. if ($null -ne $JobParams) { @@ -3171,6 +3202,13 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it. Nothing was started.") } } + # -contains, not ContainsKey: case-insensitive like binding, even + # for a case-sensitive [hashtable]::new() job. + foreach ($mandatoryParam in $mandatoryParams) { + if (@($JobParams[$jobIndex].Keys) -notcontains $mandatoryParam) { + throw ("Invoke-PkRunspaceJobs: job " + $jobIndex + " doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory. Nothing was started.") + } + } if ($checkSafety) { $sbKey = Get-PkScriptBlockJobKey -Job $JobParams[$jobIndex] if ($null -ne $sbKey) { throw ((Get-PkScriptBlockJobMessage -Job ("job " + $jobIndex) -Key $sbKey) + " Nothing was started.") } @@ -3281,6 +3319,11 @@ function Invoke-PkRunspaceJobs { throw ("Invoke-PkRunspaceJobs: a job supplies '" + $key + "', but -WorkerScript's param() block doesn't declare it.") } } + foreach ($mandatoryParam in $mandatoryParams) { + if (@($nextJob.Keys) -notcontains $mandatoryParam) { # case-insensitive, as for -JobParams above + throw ("Invoke-PkRunspaceJobs: a job from -NextJobProvider doesn't supply '" + $mandatoryParam + "', which -WorkerScript's param() block makes mandatory.") + } + } if ($checkSafety -and $null -eq $JobParams) { $sbKey = Get-PkScriptBlockJobKey -Job $nextJob if ($null -ne $sbKey) { throw (Get-PkScriptBlockJobMessage -Job "a job from -NextJobProvider" -Key $sbKey) } diff --git a/README.md b/README.md index f019c36..bd9aab8 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Programs run in your current location (`Get-Location`) unless a command says oth `-TimeoutSeconds 600` kills any program still running after 10 minutes. Its result has `TimedOut = $true`, still goes through `-OnComplete`, and the rest of the batch carries on. The default, 0, means no limit. Any programs it started are killed with it. It can be fractional. A command's own `TimeoutSeconds` key overrides this for that command; 0 opts it out of the limit. A bad value there gives that command a `StartError`. -Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). +Output is decoded in the console's code page unless you say otherwise, which is right for Windows' own tools. Many cross-platform tools write UTF-8 instead, and their non-ASCII output (accented or Japanese file names, say) comes back garbled. Pass `-OutputEncoding 'utf-8'` for the whole batch, or put an `OutputEncoding` key on one command. It also takes a code page number (`65001`, `437`, or the same as a string, as read from a CSV or config file) or a `[System.Text.Encoding]`, and the same option exists on `Invoke-PkProcessWithTimeout` and `Start-PkProcess` (and as a job key for the dispatcher's default starter). `Invoke-PkRun { ... }` is what makes Ctrl+C graceful. Whatever the block outputs is passed through, so `$batch` above gets the batch result. It also stops the PC idle-sleeping until the run ends (`-KeepAwake $false` to opt out); the display can still turn off. Without `Invoke-PkRun` the engines still work, but Ctrl+C kills the script the normal way. See "Abort" below for what it means for your own code inside the braces. @@ -186,7 +186,7 @@ $out = Invoke-PkRun { Invoke-PkRunspaceJobs @jobsParams } Each worker runs in its own runspace: a separate PowerShell session inside the same process, with its own variables and functions. That separation is what makes running them at the same time safe, and also why a worker can't see your script's variables or functions unless you hand them over. -- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key. A key it doesn't declare is an error; with a fixed `-JobParams` list it's caught before any worker starts. +- Each job hashtable's keys become the worker's parameters. The worker's `param()` block must declare every key, and every job must supply each parameter it makes `[Parameter(Mandatory)]` (`CancelToken` aside, which is filled in). Either mistake is an error; with a fixed `-JobParams` list it's caught before any worker starts, and a `-NextJobProvider` job is checked as it arrives. A worker with parameter sets isn't checked for mandatory ones, since what it needs depends on the set. - If the worker declares a `$CancelToken` parameter, it gets the run's cancellation token. - `-JobParams` takes a list or array of job hashtables. A single hashtable works too, so a list that happens to hold one job is fine. - A worker runspace starts empty. Name every one of your functions it calls in `-SharedFunctionNames`. The kit's own worker functions are always there, and cmdlets and functions from installed modules load on their own. A module you imported by path isn't installed, so list its functions too (or put it on `$env:PSModulePath`).