diff options
Diffstat (limited to 'lib/ansible/executor/powershell')
| -rw-r--r-- | lib/ansible/executor/powershell/__init__.py | 0 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/async_watchdog.ps1 | 117 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/async_wrapper.ps1 | 174 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/become_wrapper.ps1 | 163 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/bootstrap_wrapper.ps1 | 13 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/coverage_wrapper.ps1 | 199 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/exec_wrapper.ps1 | 237 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/module_manifest.py | 402 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/module_powershell_wrapper.ps1 | 75 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/module_script_wrapper.ps1 | 22 | ||||
| -rw-r--r-- | lib/ansible/executor/powershell/module_wrapper.ps1 | 226 |
11 files changed, 1628 insertions, 0 deletions
diff --git a/lib/ansible/executor/powershell/__init__.py b/lib/ansible/executor/powershell/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/ansible/executor/powershell/__init__.py diff --git a/lib/ansible/executor/powershell/async_watchdog.ps1 b/lib/ansible/executor/powershell/async_watchdog.ps1 new file mode 100644 index 0000000..c2138e3 --- /dev/null +++ b/lib/ansible/executor/powershell/async_watchdog.ps1 @@ -0,0 +1,117 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +# help with debugging errors as we don't have visibility of this running process +trap { + $watchdog_path = "$($env:TEMP)\ansible-async-watchdog-error-$(Get-Date -Format "yyyy-MM-ddTHH-mm-ss.ffffZ").txt" + $error_msg = "Error while running the async exec wrapper`r`n$(Format-AnsibleException -ErrorRecord $_)" + Set-Content -Path $watchdog_path -Value $error_msg + break +} + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting async_watchdog" "async_watchdog" + +# pop 0th action as entrypoint +$payload.actions = $payload.actions[1..99] + +$actions = $Payload.actions +$entrypoint = $payload.($actions[0]) +$entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($entrypoint)) + +$resultfile_path = $payload.async_results_path +$max_exec_time_sec = $payload.async_timeout_sec + +Write-AnsibleLog "INFO - deserializing existing result file args at: '$resultfile_path'" "async_watchdog" +if (-not (Test-Path -Path $resultfile_path)) { + $msg = "result file at '$resultfile_path' does not exist" + Write-AnsibleLog "ERROR - $msg" "async_watchdog" + throw $msg +} +$result_json = Get-Content -Path $resultfile_path -Raw +Write-AnsibleLog "INFO - result file json is: $result_json" "async_watchdog" +$result = ConvertFrom-AnsibleJson -InputObject $result_json + +Write-AnsibleLog "INFO - creating async runspace" "async_watchdog" +$rs = [RunspaceFactory]::CreateRunspace() +$rs.Open() + +Write-AnsibleLog "INFO - creating async PowerShell pipeline" "async_watchdog" +$ps = [PowerShell]::Create() +$ps.Runspace = $rs + +# these functions are set in exec_wrapper +Write-AnsibleLog "INFO - adding global functions to PowerShell pipeline script" "async_watchdog" +$ps.AddScript($script:common_functions).AddStatement() > $null +$ps.AddScript($script:wrapper_functions).AddStatement() > $null +$function_params = @{ + Name = "common_functions" + Value = $script:common_functions + Scope = "script" +} +$ps.AddCommand("Set-Variable").AddParameters($function_params).AddStatement() > $null + +Write-AnsibleLog "INFO - adding $($actions[0]) to PowerShell pipeline script" "async_watchdog" +$ps.AddScript($entrypoint).AddArgument($payload) > $null + +Write-AnsibleLog "INFO - async job start, calling BeginInvoke()" "async_watchdog" +$job_async_result = $ps.BeginInvoke() + +Write-AnsibleLog "INFO - waiting '$max_exec_time_sec' seconds for async job to complete" "async_watchdog" +$job_async_result.AsyncWaitHandle.WaitOne($max_exec_time_sec * 1000) > $null +$result.finished = 1 + +if ($job_async_result.IsCompleted) { + Write-AnsibleLog "INFO - async job completed, calling EndInvoke()" "async_watchdog" + + $job_output = $ps.EndInvoke($job_async_result) + $job_error = $ps.Streams.Error + + Write-AnsibleLog "INFO - raw module stdout:`r`n$($job_output | Out-String)" "async_watchdog" + if ($job_error) { + Write-AnsibleLog "WARN - raw module stderr:`r`n$($job_error | Out-String)" "async_watchdog" + } + + # write success/output/error to result object + # TODO: cleanse leading/trailing junk + try { + Write-AnsibleLog "INFO - deserializing Ansible stdout" "async_watchdog" + $module_result = ConvertFrom-AnsibleJson -InputObject $job_output + # TODO: check for conflicting keys + $result = $result + $module_result + } + catch { + $result.failed = $true + $result.msg = "failed to parse module output: $($_.Exception.Message)" + # return output back to Ansible to help with debugging errors + $result.stdout = $job_output | Out-String + $result.stderr = $job_error | Out-String + } + + $result_json = ConvertTo-Json -InputObject $result -Depth 99 -Compress + Set-Content -Path $resultfile_path -Value $result_json + + Write-AnsibleLog "INFO - wrote output to $resultfile_path" "async_watchdog" +} +else { + Write-AnsibleLog "ERROR - reached timeout on async job, stopping job" "async_watchdog" + $ps.BeginStop($null, $null) > $null # best effort stop + + # write timeout to result object + $result.failed = $true + $result.msg = "timed out waiting for module completion" + $result_json = ConvertTo-Json -InputObject $result -Depth 99 -Compress + Set-Content -Path $resultfile_path -Value $result_json + + Write-AnsibleLog "INFO - wrote timeout to '$resultfile_path'" "async_watchdog" +} + +# in the case of a hung pipeline, this will cause the process to stay alive until it's un-hung... +#$rs.Close() | Out-Null + +Write-AnsibleLog "INFO - ending async_watchdog" "async_watchdog" diff --git a/lib/ansible/executor/powershell/async_wrapper.ps1 b/lib/ansible/executor/powershell/async_wrapper.ps1 new file mode 100644 index 0000000..0cd640f --- /dev/null +++ b/lib/ansible/executor/powershell/async_wrapper.ps1 @@ -0,0 +1,174 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting async_wrapper" "async_wrapper" + +if (-not $Payload.environment.ContainsKey("ANSIBLE_ASYNC_DIR")) { + Write-AnsibleError -Message "internal error: the environment variable ANSIBLE_ASYNC_DIR is not set and is required for an async task" + $host.SetShouldExit(1) + return +} +$async_dir = [System.Environment]::ExpandEnvironmentVariables($Payload.environment.ANSIBLE_ASYNC_DIR) + +# calculate the result path so we can include it in the worker payload +$jid = $Payload.async_jid +$local_jid = $jid + "." + $pid + +$results_path = [System.IO.Path]::Combine($async_dir, $local_jid) + +Write-AnsibleLog "INFO - creating async results path at '$results_path'" "async_wrapper" + +$Payload.async_results_path = $results_path +[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($results_path)) > $null + +# we use Win32_Process to escape the current process job, CreateProcess with a +# breakaway flag won't work for psrp as the psrp process does not have breakaway +# rights. Unfortunately we can't read/write to the spawned process as we can't +# inherit the handles. We use a locked down named pipe to send the exec_wrapper +# payload. Anonymous pipes won't work as the spawned process will not be a child +# of the current one and will not be able to inherit the handles + +# pop the async_wrapper action so we don't get stuck in a loop and create new +# exec_wrapper for our async process +$Payload.actions = $Payload.actions[1..99] +$payload_json = ConvertTo-Json -InputObject $Payload -Depth 99 -Compress + +# +$exec_wrapper = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.exec_wrapper)) +$exec_wrapper += "`0`0`0`0" + $payload_json +$payload_bytes = [System.Text.Encoding]::UTF8.GetBytes($exec_wrapper) +$pipe_name = "ansible-async-$jid-$([guid]::NewGuid())" + +# template the async process command line with the payload details +$bootstrap_wrapper = { + # help with debugging errors as we loose visibility of the process output + # from here on + trap { + $wrapper_path = "$($env:TEMP)\ansible-async-wrapper-error-$(Get-Date -Format "yyyy-MM-ddTHH-mm-ss.ffffZ").txt" + $error_msg = "Error while running the async exec wrapper`r`n$($_ | Out-String)`r`n$($_.ScriptStackTrace)" + Set-Content -Path $wrapper_path -Value $error_msg + break + } + + &chcp.com 65001 > $null + + # store the pipe name and no. of bytes to read, these are populated before + # before the process is created - do not remove or changed + $pipe_name = "" + $bytes_length = 0 + + $input_bytes = New-Object -TypeName byte[] -ArgumentList $bytes_length + $pipe = New-Object -TypeName System.IO.Pipes.NamedPipeClientStream -ArgumentList @( + ".", # localhost + $pipe_name, + [System.IO.Pipes.PipeDirection]::In, + [System.IO.Pipes.PipeOptions]::None, + [System.Security.Principal.TokenImpersonationLevel]::Anonymous + ) + try { + $pipe.Connect() + $pipe.Read($input_bytes, 0, $bytes_length) > $null + } + finally { + $pipe.Close() + } + $exec = [System.Text.Encoding]::UTF8.GetString($input_bytes) + $exec_parts = $exec.Split(@("`0`0`0`0"), 2, [StringSplitOptions]::RemoveEmptyEntries) + Set-Variable -Name json_raw -Value $exec_parts[1] + $exec = [ScriptBlock]::Create($exec_parts[0]) + &$exec +} + +$bootstrap_wrapper = $bootstrap_wrapper.ToString().Replace('$pipe_name = ""', "`$pipe_name = `"$pipe_name`"") +$bootstrap_wrapper = $bootstrap_wrapper.Replace('$bytes_length = 0', "`$bytes_length = $($payload_bytes.Count)") +$encoded_command = [System.Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($bootstrap_wrapper)) +$pwsh_path = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" +$exec_args = "`"$pwsh_path`" -NonInteractive -NoProfile -ExecutionPolicy Bypass -EncodedCommand $encoded_command" + +# create a named pipe that is set to allow only the current user read access +$current_user = ([Security.Principal.WindowsIdentity]::GetCurrent()).User +$pipe_sec = New-Object -TypeName System.IO.Pipes.PipeSecurity +$pipe_ar = New-Object -TypeName System.IO.Pipes.PipeAccessRule -ArgumentList @( + $current_user, + [System.IO.Pipes.PipeAccessRights]::Read, + [System.Security.AccessControl.AccessControlType]::Allow +) +$pipe_sec.AddAccessRule($pipe_ar) + +Write-AnsibleLog "INFO - creating named pipe '$pipe_name'" "async_wrapper" +$pipe = New-Object -TypeName System.IO.Pipes.NamedPipeServerStream -ArgumentList @( + $pipe_name, + [System.IO.Pipes.PipeDirection]::Out, + 1, + [System.IO.Pipes.PipeTransmissionMode]::Byte, + [System.IO.Pipes.PipeOptions]::Asynchronous, + 0, + 0, + $pipe_sec +) + +try { + Write-AnsibleLog "INFO - creating async process '$exec_args'" "async_wrapper" + $process = Invoke-CimMethod -ClassName Win32_Process -Name Create -Arguments @{CommandLine = $exec_args } + $rc = $process.ReturnValue + + Write-AnsibleLog "INFO - return value from async process exec: $rc" "async_wrapper" + if ($rc -ne 0) { + $error_msg = switch ($rc) { + 2 { "Access denied" } + 3 { "Insufficient privilege" } + 8 { "Unknown failure" } + 9 { "Path not found" } + 21 { "Invalid parameter" } + default { "Other" } + } + throw "Failed to start async process: $rc ($error_msg)" + } + $watchdog_pid = $process.ProcessId + Write-AnsibleLog "INFO - created async process PID: $watchdog_pid" "async_wrapper" + + # populate initial results before we send the async data to avoid result race + $result = @{ + started = 1; + finished = 0; + results_file = $results_path; + ansible_job_id = $local_jid; + _ansible_suppress_tmpdir_delete = $true; + ansible_async_watchdog_pid = $watchdog_pid + } + + Write-AnsibleLog "INFO - writing initial async results to '$results_path'" "async_wrapper" + $result_json = ConvertTo-Json -InputObject $result -Depth 99 -Compress + Set-Content $results_path -Value $result_json + + $np_timeout = $Payload.async_startup_timeout * 1000 + Write-AnsibleLog "INFO - waiting for async process to connect to named pipe for $np_timeout milliseconds" "async_wrapper" + $wait_async = $pipe.BeginWaitForConnection($null, $null) + $wait_async.AsyncWaitHandle.WaitOne($np_timeout) > $null + if (-not $wait_async.IsCompleted) { + $msg = "Ansible encountered a timeout while waiting for the async task to start and connect to the named" + $msg += "pipe. This can be affected by the performance of the target - you can increase this timeout using" + $msg += "WIN_ASYNC_STARTUP_TIMEOUT or just for this host using the win_async_startup_timeout hostvar if " + $msg += "this keeps happening." + throw $msg + } + $pipe.EndWaitForConnection($wait_async) + + Write-AnsibleLog "INFO - writing exec_wrapper and payload to async process" "async_wrapper" + $pipe.Write($payload_bytes, 0, $payload_bytes.Count) + $pipe.Flush() + $pipe.WaitForPipeDrain() +} +finally { + $pipe.Close() +} + +Write-AnsibleLog "INFO - outputting initial async result: $result_json" "async_wrapper" +Write-Output -InputObject $result_json +Write-AnsibleLog "INFO - ending async_wrapper" "async_wrapper" diff --git a/lib/ansible/executor/powershell/become_wrapper.ps1 b/lib/ansible/executor/powershell/become_wrapper.ps1 new file mode 100644 index 0000000..f40e265 --- /dev/null +++ b/lib/ansible/executor/powershell/become_wrapper.ps1 @@ -0,0 +1,163 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +#Requires -Module Ansible.ModuleUtils.AddType +#AnsibleRequires -CSharpUtil Ansible.AccessToken +#AnsibleRequires -CSharpUtil Ansible.Become + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting become_wrapper" "become_wrapper" + +Function Get-EnumValue($enum, $flag_type, $value) { + $raw_enum_value = $value.Replace('_', '') + try { + $enum_value = [Enum]::Parse($enum, $raw_enum_value, $true) + } + catch [System.ArgumentException] { + $valid_options = [Enum]::GetNames($enum) | ForEach-Object -Process { + (($_ -creplace "(.)([A-Z][a-z]+)", '$1_$2') -creplace "([a-z0-9])([A-Z])", '$1_$2').ToString().ToLower() + } + throw "become_flags $flag_type value '$value' is not valid, valid values are: $($valid_options -join ", ")" + } + return $enum_value +} + +Function Get-BecomeFlag($flags) { + $logon_type = [Ansible.AccessToken.LogonType]::Interactive + $logon_flags = [Ansible.Become.LogonFlags]::WithProfile + + if ($null -eq $flags -or $flags -eq "") { + $flag_split = @() + } + elseif ($flags -is [string]) { + $flag_split = $flags.Split(" ") + } + else { + throw "become_flags must be a string, was $($flags.GetType())" + } + + foreach ($flag in $flag_split) { + $split = $flag.Split("=") + if ($split.Count -ne 2) { + throw "become_flags entry '$flag' is in an invalid format, must be a key=value pair" + } + $flag_key = $split[0] + $flag_value = $split[1] + if ($flag_key -eq "logon_type") { + $enum_details = @{ + enum = [Ansible.AccessToken.LogonType] + flag_type = $flag_key + value = $flag_value + } + $logon_type = Get-EnumValue @enum_details + } + elseif ($flag_key -eq "logon_flags") { + $logon_flag_values = $flag_value.Split(",") + $logon_flags = 0 -as [Ansible.Become.LogonFlags] + foreach ($logon_flag_value in $logon_flag_values) { + if ($logon_flag_value -eq "") { + continue + } + $enum_details = @{ + enum = [Ansible.Become.LogonFlags] + flag_type = $flag_key + value = $logon_flag_value + } + $logon_flag = Get-EnumValue @enum_details + $logon_flags = $logon_flags -bor $logon_flag + } + } + else { + throw "become_flags key '$flag_key' is not a valid runas flag, must be 'logon_type' or 'logon_flags'" + } + } + + return $logon_type, [Ansible.Become.LogonFlags]$logon_flags +} + +Write-AnsibleLog "INFO - loading C# become code" "become_wrapper" +$add_type_b64 = $Payload.powershell_modules["Ansible.ModuleUtils.AddType"] +$add_type = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($add_type_b64)) +New-Module -Name Ansible.ModuleUtils.AddType -ScriptBlock ([ScriptBlock]::Create($add_type)) | Import-Module > $null + +$new_tmp = [System.Environment]::ExpandEnvironmentVariables($Payload.module_args["_ansible_remote_tmp"]) +$access_def = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.csharp_utils["Ansible.AccessToken"])) +$become_def = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.csharp_utils["Ansible.Become"])) +$process_def = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.csharp_utils["Ansible.Process"])) +Add-CSharpType -References $access_def, $become_def, $process_def -TempPath $new_tmp -IncludeDebugInfo + +$username = $Payload.become_user +$password = $Payload.become_password +# We need to set password to the value of NullString so a null password is preserved when crossing the .NET +# boundary. If we pass $null it will automatically be converted to "" and we need to keep the distinction for +# accounts that don't have a password and when someone wants to become without knowing the password. +if ($null -eq $password) { + $password = [NullString]::Value +} + +try { + $logon_type, $logon_flags = Get-BecomeFlag -flags $Payload.become_flags +} +catch { + Write-AnsibleError -Message "internal error: failed to parse become_flags '$($Payload.become_flags)'" -ErrorRecord $_ + $host.SetShouldExit(1) + return +} +Write-AnsibleLog "INFO - parsed become input, user: '$username', type: '$logon_type', flags: '$logon_flags'" "become_wrapper" + +# NB: CreateProcessWithTokenW commandline maxes out at 1024 chars, must +# bootstrap via small wrapper which contains the exec_wrapper passed through the +# stdin pipe. Cannot use 'powershell -' as the $ErrorActionPreference is always +# set to Stop and cannot be changed. Also need to split the payload from the wrapper to prevent potentially +# sensitive content from being logged by the scriptblock logger. +$bootstrap_wrapper = { + &chcp.com 65001 > $null + $exec_wrapper_str = [System.Console]::In.ReadToEnd() + $split_parts = $exec_wrapper_str.Split(@("`0`0`0`0"), 2, [StringSplitOptions]::RemoveEmptyEntries) + Set-Variable -Name json_raw -Value $split_parts[1] + $exec_wrapper = [ScriptBlock]::Create($split_parts[0]) + &$exec_wrapper +} +$exec_command = [System.Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($bootstrap_wrapper.ToString())) +$lp_command_line = "powershell.exe -NonInteractive -NoProfile -ExecutionPolicy Bypass -EncodedCommand $exec_command" +$lp_current_directory = $env:SystemRoot # TODO: should this be set to the become user's profile dir? + +# pop the become_wrapper action so we don't get stuck in a loop +$Payload.actions = $Payload.actions[1..99] +# we want the output from the exec_wrapper to be base64 encoded to preserve unicode chars +$Payload.encoded_output = $true + +$payload_json = ConvertTo-Json -InputObject $Payload -Depth 99 -Compress +# delimit the payload JSON from the wrapper to keep sensitive contents out of scriptblocks (which can be logged) +$exec_wrapper = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.exec_wrapper)) +$exec_wrapper += "`0`0`0`0" + $payload_json + +try { + Write-AnsibleLog "INFO - starting become process '$lp_command_line'" "become_wrapper" + $result = [Ansible.Become.BecomeUtil]::CreateProcessAsUser($username, $password, $logon_flags, $logon_type, + $null, $lp_command_line, $lp_current_directory, $null, $exec_wrapper) + Write-AnsibleLog "INFO - become process complete with rc: $($result.ExitCode)" "become_wrapper" + $stdout = $result.StandardOut + try { + $stdout = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($stdout)) + } + catch [FormatException] { + # output wasn't Base64, ignore as it may contain an error message we want to pass to Ansible + Write-AnsibleLog "WARN - become process stdout was not base64 encoded as expected: $stdout" + } + + $host.UI.WriteLine($stdout) + $host.UI.WriteErrorLine($result.StandardError.Trim()) + $host.SetShouldExit($result.ExitCode) +} +catch { + Write-AnsibleError -Message "internal error: failed to become user '$username'" -ErrorRecord $_ + $host.SetShouldExit(1) +} + +Write-AnsibleLog "INFO - ending become_wrapper" "become_wrapper" diff --git a/lib/ansible/executor/powershell/bootstrap_wrapper.ps1 b/lib/ansible/executor/powershell/bootstrap_wrapper.ps1 new file mode 100644 index 0000000..cdba80c --- /dev/null +++ b/lib/ansible/executor/powershell/bootstrap_wrapper.ps1 @@ -0,0 +1,13 @@ +&chcp.com 65001 > $null + +if ($PSVersionTable.PSVersion -lt [Version]"3.0") { + '{"failed":true,"msg":"Ansible requires PowerShell v3.0 or newer"}' + exit 1 +} + +$exec_wrapper_str = $input | Out-String +$split_parts = $exec_wrapper_str.Split(@("`0`0`0`0"), 2, [StringSplitOptions]::RemoveEmptyEntries) +If (-not $split_parts.Length -eq 2) { throw "invalid payload" } +Set-Variable -Name json_raw -Value $split_parts[1] +$exec_wrapper = [ScriptBlock]::Create($split_parts[0]) +&$exec_wrapper diff --git a/lib/ansible/executor/powershell/coverage_wrapper.ps1 b/lib/ansible/executor/powershell/coverage_wrapper.ps1 new file mode 100644 index 0000000..26cbe66 --- /dev/null +++ b/lib/ansible/executor/powershell/coverage_wrapper.ps1 @@ -0,0 +1,199 @@ +# (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +#AnsibleRequires -Wrapper module_wrapper + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting coverage_wrapper" "coverage_wrapper" + +# Required to be set for psrp to we can set a breakpoint in the remote runspace +if ($PSVersionTable.PSVersion -ge [Version]'4.0') { + $host.Runspace.Debugger.SetDebugMode([System.Management.Automation.DebugModes]::RemoteScript) +} + +Function New-CoverageBreakpoint { + Param ( + [String]$Path, + [ScriptBlock]$Code, + [String]$AnsiblePath + ) + + # It is quicker to pass in the code as a string instead of calling ParseFile as we already know the contents + $predicate = { + $args[0] -is [System.Management.Automation.Language.CommandBaseAst] + } + $script_cmds = $Code.Ast.FindAll($predicate, $true) + + # Create an object that tracks the Ansible path of the file and the breakpoints that have been set in it + $info = [PSCustomObject]@{ + Path = $AnsiblePath + Breakpoints = [System.Collections.Generic.List`1[System.Management.Automation.Breakpoint]]@() + } + + # Keep track of lines that are already scanned. PowerShell can contains multiple commands in 1 line + $scanned_lines = [System.Collections.Generic.HashSet`1[System.Int32]]@() + foreach ($cmd in $script_cmds) { + if (-not $scanned_lines.Add($cmd.Extent.StartLineNumber)) { + continue + } + + # Do not add any -Action value, even if it is $null or {}. Doing so will balloon the runtime. + $params = @{ + Script = $Path + Line = $cmd.Extent.StartLineNumber + Column = $cmd.Extent.StartColumnNumber + } + $info.Breakpoints.Add((Set-PSBreakpoint @params)) + } + + $info +} + +Function Compare-PathFilterPattern { + Param ( + [String[]]$Patterns, + [String]$Path + ) + + foreach ($pattern in $Patterns) { + if ($Path -like $pattern) { + return $true + } + } + return $false +} + +$module_name = $Payload.module_args["_ansible_module_name"] +Write-AnsibleLog "INFO - building coverage payload for '$module_name'" "coverage_wrapper" + +# A PS Breakpoint needs an actual path to work properly, we create a temp directory that will store the module and +# module_util code during execution +$temp_path = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath "ansible-coverage-$([System.IO.Path]::GetRandomFileName())" +Write-AnsibleLog "INFO - Creating temp path for coverage files '$temp_path'" "coverage_wrapper" +New-Item -Path $temp_path -ItemType Directory > $null +$breakpoint_info = [System.Collections.Generic.List`1[PSObject]]@() + +# Ensures we create files with UTF-8 encoding and a BOM. This is critical to force the powershell engine to read files +# as UTF-8 and not as the system's codepage. +$file_encoding = 'UTF8' + +try { + $scripts = [System.Collections.Generic.List`1[System.Object]]@($script:common_functions) + + $coverage_path_filter = $Payload.coverage.path_filter.Split(":", [StringSplitOptions]::RemoveEmptyEntries) + + # We need to track what utils have already been added to the script for loading. This is because the load + # order is important and can have module_utils that rely on other utils. + $loaded_utils = [System.Collections.Generic.HashSet`1[System.String]]@() + $parse_util = { + $util_name = $args[0] + if (-not $loaded_utils.Add($util_name)) { + return + } + + $util_code = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.powershell_modules.$util_name)) + $util_sb = [ScriptBlock]::Create($util_code) + $util_path = Join-Path -Path $temp_path -ChildPath "$($util_name).psm1" + + Write-AnsibleLog "INFO - Outputting module_util $util_name to temp file '$util_path'" "coverage_wrapper" + Set-Content -LiteralPath $util_path -Value $util_code -Encoding $file_encoding + + $ansible_path = $Payload.coverage.module_util_paths.$util_name + if ((Compare-PathFilterPattern -Patterns $coverage_path_filter -Path $ansible_path)) { + $cov_params = @{ + Path = $util_path + Code = $util_sb + AnsiblePath = $ansible_path + } + $breakpoints = New-CoverageBreakpoint @cov_params + $breakpoint_info.Add($breakpoints) + } + + if ($null -ne $util_sb.Ast.ScriptRequirements) { + foreach ($required_util in $util_sb.Ast.ScriptRequirements.RequiredModules) { + &$parse_util $required_util.Name + } + } + Write-AnsibleLog "INFO - Adding util $util_name to scripts to run" "coverage_wrapper" + $scripts.Add("Import-Module -Name '$util_path'") + } + foreach ($util in $Payload.powershell_modules.Keys) { + &$parse_util $util + } + + $module = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.module_entry)) + $module_path = Join-Path -Path $temp_path -ChildPath "$($module_name).ps1" + Write-AnsibleLog "INFO - Ouputting module $module_name to temp file '$module_path'" "coverage_wrapper" + Set-Content -LiteralPath $module_path -Value $module -Encoding $file_encoding + $scripts.Add($module_path) + + $ansible_path = $Payload.coverage.module_path + if ((Compare-PathFilterPattern -Patterns $coverage_path_filter -Path $ansible_path)) { + $cov_params = @{ + Path = $module_path + Code = [ScriptBlock]::Create($module) + AnsiblePath = $Payload.coverage.module_path + } + $breakpoints = New-CoverageBreakpoint @cov_params + $breakpoint_info.Add($breakpoints) + } + + $variables = [System.Collections.ArrayList]@(@{ Name = "complex_args"; Value = $Payload.module_args; Scope = "Global" }) + $entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($payload.module_wrapper)) + $entrypoint = [ScriptBlock]::Create($entrypoint) + + $params = @{ + Scripts = $scripts + Variables = $variables + Environment = $Payload.environment + ModuleName = $module_name + } + if ($breakpoint_info) { + $params.Breakpoints = $breakpoint_info.Breakpoints + } + + try { + &$entrypoint @params + } + finally { + # Processing here is kept to an absolute minimum to make sure each task runtime is kept as small as + # possible. Once all the tests have been run ansible-test will collect this info and process it locally in + # one go. + Write-AnsibleLog "INFO - Creating coverage result output" "coverage_wrapper" + $coverage_info = @{} + foreach ($info in $breakpoint_info) { + $coverage_info.($info.Path) = $info.Breakpoints | Select-Object -Property Line, HitCount + } + + # The coverage.output value is a filename set by the Ansible controller. We append some more remote side + # info to the filename to make it unique and identify the remote host a bit more. + $ps_version = "$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + $coverage_output_path = "$($Payload.coverage.output)=powershell-$ps_version=coverage.$($env:COMPUTERNAME).$PID.$(Get-Random)" + $code_cov_json = ConvertTo-Json -InputObject $coverage_info -Compress + + Write-AnsibleLog "INFO - Outputting coverage json to '$coverage_output_path'" "coverage_wrapper" + # Ansible controller expects these files to be UTF-8 without a BOM, use .NET for this. + $utf8_no_bom = New-Object -TypeName System.Text.UTF8Encoding -ArgumentList $false + [System.IO.File]::WriteAllbytes($coverage_output_path, $utf8_no_bom.GetBytes($code_cov_json)) + } +} +finally { + try { + if ($breakpoint_info) { + foreach ($b in $breakpoint_info.Breakpoints) { + Remove-PSBreakpoint -Breakpoint $b + } + } + } + finally { + Write-AnsibleLog "INFO - Remove temp coverage folder '$temp_path'" "coverage_wrapper" + Remove-Item -LiteralPath $temp_path -Force -Recurse + } +} + +Write-AnsibleLog "INFO - ending coverage_wrapper" "coverage_wrapper" diff --git a/lib/ansible/executor/powershell/exec_wrapper.ps1 b/lib/ansible/executor/powershell/exec_wrapper.ps1 new file mode 100644 index 0000000..0f97bdf --- /dev/null +++ b/lib/ansible/executor/powershell/exec_wrapper.ps1 @@ -0,0 +1,237 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +begin { + $DebugPreference = "Continue" + $ProgressPreference = "SilentlyContinue" + $ErrorActionPreference = "Stop" + Set-StrictMode -Version 2 + + # common functions that are loaded in exec and module context, this is set + # as a script scoped variable so async_watchdog and module_wrapper can + # access the functions when creating their Runspaces + $script:common_functions = { + Function ConvertFrom-AnsibleJson { + <# + .SYNOPSIS + Converts a JSON string to a Hashtable/Array in the fastest way + possible. Unfortunately ConvertFrom-Json is still faster but outputs + a PSCustomObject which is combersone for module consumption. + + .PARAMETER InputObject + [String] The JSON string to deserialize. + #> + param( + [Parameter(Mandatory = $true, Position = 0)][String]$InputObject + ) + + # we can use -AsHashtable to get PowerShell to convert the JSON to + # a Hashtable and not a PSCustomObject. This was added in PowerShell + # 6.0, fall back to a manual conversion for older versions + $cmdlet = Get-Command -Name ConvertFrom-Json -CommandType Cmdlet + if ("AsHashtable" -in $cmdlet.Parameters.Keys) { + return , (ConvertFrom-Json -InputObject $InputObject -AsHashtable) + } + else { + # get the PSCustomObject and then manually convert from there + $raw_obj = ConvertFrom-Json -InputObject $InputObject + + Function ConvertTo-Hashtable { + param($InputObject) + + if ($null -eq $InputObject) { + return $null + } + + if ($InputObject -is [PSCustomObject]) { + $new_value = @{} + foreach ($prop in $InputObject.PSObject.Properties.GetEnumerator()) { + $new_value.($prop.Name) = (ConvertTo-Hashtable -InputObject $prop.Value) + } + return , $new_value + } + elseif ($InputObject -is [Array]) { + $new_value = [System.Collections.ArrayList]@() + foreach ($val in $InputObject) { + $new_value.Add((ConvertTo-Hashtable -InputObject $val)) > $null + } + return , $new_value.ToArray() + } + else { + return , $InputObject + } + } + return , (ConvertTo-Hashtable -InputObject $raw_obj) + } + } + + Function Format-AnsibleException { + <# + .SYNOPSIS + Formats a PowerShell ErrorRecord to a string that's fit for human + consumption. + + .NOTES + Using Out-String can give us the first part of the exception but it + also wraps the messages at 80 chars which is not ideal. We also + append the ScriptStackTrace and the .NET StackTrace if present. + #> + param([System.Management.Automation.ErrorRecord]$ErrorRecord) + + $exception = @" +$($ErrorRecord.ToString()) +$($ErrorRecord.InvocationInfo.PositionMessage) + + CategoryInfo : $($ErrorRecord.CategoryInfo.ToString()) + + FullyQualifiedErrorId : $($ErrorRecord.FullyQualifiedErrorId.ToString()) +"@ + # module_common strip comments and empty newlines, need to manually + # add a preceding newline using `r`n + $exception += "`r`n`r`nScriptStackTrace:`r`n$($ErrorRecord.ScriptStackTrace)`r`n" + + # exceptions from C# will also have a StackTrace which we + # append if found + if ($null -ne $ErrorRecord.Exception.StackTrace) { + $exception += "`r`n$($ErrorRecord.Exception.ToString())" + } + + return $exception + } + } + .$common_functions + + # common wrapper functions used in the exec wrappers, this is defined in a + # script scoped variable so async_watchdog can pass them into the async job + $script:wrapper_functions = { + Function Write-AnsibleError { + <# + .SYNOPSIS + Writes an error message to a JSON string in the format that Ansible + understands. Also optionally adds an exception record if the + ErrorRecord is passed through. + #> + param( + [Parameter(Mandatory = $true)][String]$Message, + [System.Management.Automation.ErrorRecord]$ErrorRecord = $null + ) + $result = @{ + msg = $Message + failed = $true + } + if ($null -ne $ErrorRecord) { + $result.msg += ": $($ErrorRecord.Exception.Message)" + $result.exception = (Format-AnsibleException -ErrorRecord $ErrorRecord) + } + Write-Output -InputObject (ConvertTo-Json -InputObject $result -Depth 99 -Compress) + } + + Function Write-AnsibleLog { + <# + .SYNOPSIS + Used as a debugging tool to log events to a file as they run in the + exec wrappers. By default this is a noop function but the $log_path + can be manually set to enable it. Manually set ANSIBLE_EXEC_DEBUG as + an env value on the Windows host that this is run on to enable. + #> + param( + [Parameter(Mandatory = $true, Position = 0)][String]$Message, + [Parameter(Position = 1)][String]$Wrapper + ) + + $log_path = $env:ANSIBLE_EXEC_DEBUG + if ($log_path) { + $log_path = [System.Environment]::ExpandEnvironmentVariables($log_path) + $parent_path = [System.IO.Path]::GetDirectoryName($log_path) + if (Test-Path -LiteralPath $parent_path -PathType Container) { + $msg = "{0:u} - {1} - {2} - " -f (Get-Date), $pid, ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) + if ($null -ne $Wrapper) { + $msg += "$Wrapper - " + } + $msg += $Message + "`r`n" + $msg_bytes = [System.Text.Encoding]::UTF8.GetBytes($msg) + + $fs = [System.IO.File]::Open($log_path, [System.IO.FileMode]::Append, + [System.IO.FileAccess]::Write, [System.IO.FileShare]::ReadWrite) + try { + $fs.Write($msg_bytes, 0, $msg_bytes.Length) + } + finally { + $fs.Close() + } + } + } + } + } + .$wrapper_functions + + # only init and stream in $json_raw if it wasn't set by the enclosing scope + if (-not $(Get-Variable "json_raw" -ErrorAction SilentlyContinue)) { + $json_raw = '' + } +} process { + $json_raw += [String]$input +} end { + Write-AnsibleLog "INFO - starting exec_wrapper" "exec_wrapper" + if (-not $json_raw) { + Write-AnsibleError -Message "internal error: no input given to PowerShell exec wrapper" + exit 1 + } + + Write-AnsibleLog "INFO - converting json raw to a payload" "exec_wrapper" + $payload = ConvertFrom-AnsibleJson -InputObject $json_raw + + # TODO: handle binary modules + # TODO: handle persistence + + if ($payload.min_os_version) { + $min_os_version = [Version]$payload.min_os_version + # Environment.OSVersion.Version is deprecated and may not return the + # right version + $actual_os_version = [Version](Get-Item -Path $env:SystemRoot\System32\kernel32.dll).VersionInfo.ProductVersion + + Write-AnsibleLog "INFO - checking if actual os version '$actual_os_version' is less than the min os version '$min_os_version'" "exec_wrapper" + if ($actual_os_version -lt $min_os_version) { + $msg = "internal error: This module cannot run on this OS as it requires a minimum version of $min_os_version, actual was $actual_os_version" + Write-AnsibleError -Message $msg + exit 1 + } + } + if ($payload.min_ps_version) { + $min_ps_version = [Version]$payload.min_ps_version + $actual_ps_version = $PSVersionTable.PSVersion + + Write-AnsibleLog "INFO - checking if actual PS version '$actual_ps_version' is less than the min PS version '$min_ps_version'" "exec_wrapper" + if ($actual_ps_version -lt $min_ps_version) { + $msg = "internal error: This module cannot run as it requires a minimum PowerShell version of $min_ps_version, actual was $actual_ps_version" + Write-AnsibleError -Message $msg + exit 1 + } + } + + # pop 0th action as entrypoint + $action = $payload.actions[0] + Write-AnsibleLog "INFO - running action $action" "exec_wrapper" + + $entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($payload.($action))) + $entrypoint = [ScriptBlock]::Create($entrypoint) + # so we preserve the formatting and don't fall prey to locale issues, some + # wrappers want the output to be in base64 form, we store the value here in + # case the wrapper changes the value when they create a payload for their + # own exec_wrapper + $encoded_output = $payload.encoded_output + + try { + $output = &$entrypoint -Payload $payload + if ($encoded_output -and $null -ne $output) { + $b64_output = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($output)) + Write-Output -InputObject $b64_output + } + else { + $output + } + } + catch { + Write-AnsibleError -Message "internal error: failed to run exec_wrapper action $action" -ErrorRecord $_ + exit 1 + } + Write-AnsibleLog "INFO - ending exec_wrapper" "exec_wrapper" +} diff --git a/lib/ansible/executor/powershell/module_manifest.py b/lib/ansible/executor/powershell/module_manifest.py new file mode 100644 index 0000000..970e848 --- /dev/null +++ b/lib/ansible/executor/powershell/module_manifest.py @@ -0,0 +1,402 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import base64 +import errno +import json +import os +import pkgutil +import random +import re + +from ansible.module_utils.compat.version import LooseVersion + +from ansible import constants as C +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.compat.importlib import import_module +from ansible.plugins.loader import ps_module_utils_loader +from ansible.utils.collection_loader import resource_from_fqcr + + +class PSModuleDepFinder(object): + + def __init__(self): + # This is also used by validate-modules to get a module's required utils in base and a collection. + self.ps_modules = dict() + self.exec_scripts = dict() + + # by defining an explicit dict of cs utils and where they are used, we + # can potentially save time by not adding the type multiple times if it + # isn't needed + self.cs_utils_wrapper = dict() + self.cs_utils_module = dict() + + self.ps_version = None + self.os_version = None + self.become = False + + self._re_cs_module = [ + # Reference C# module_util in another C# util, this must always be the fully qualified name. + # 'using ansible_collections.{namespace}.{collection}.plugins.module_utils.{name}' + re.compile(to_bytes(r'(?i)^using\s((Ansible\..+)|' + r'(ansible_collections\.\w+\.\w+\.plugins\.module_utils\.[\w\.]+));\s*$')), + ] + + self._re_cs_in_ps_module = [ + # Reference C# module_util in a PowerShell module + # '#AnsibleRequires -CSharpUtil Ansible.{name}' + # '#AnsibleRequires -CSharpUtil ansible_collections.{namespace}.{collection}.plugins.module_utils.{name}' + # '#AnsibleRequires -CSharpUtil ..module_utils.{name}' + # Can have '-Optional' at the end to denote the util is optional + re.compile(to_bytes(r'(?i)^#\s*ansiblerequires\s+-csharputil\s+((Ansible\.[\w\.]+)|' + r'(ansible_collections\.\w+\.\w+\.plugins\.module_utils\.[\w\.]+)|' + r'(\.[\w\.]+))(?P<optional>\s+-Optional){0,1}')), + ] + + self._re_ps_module = [ + # Original way of referencing a builtin module_util + # '#Requires -Module Ansible.ModuleUtils.{name} + re.compile(to_bytes(r'(?i)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)')), + # New way of referencing a builtin and collection module_util + # '#AnsibleRequires -PowerShell Ansible.ModuleUtils.{name}' + # '#AnsibleRequires -PowerShell ansible_collections.{namespace}.{collection}.plugins.module_utils.{name}' + # '#AnsibleRequires -PowerShell ..module_utils.{name}' + # Can have '-Optional' at the end to denote the util is optional + re.compile(to_bytes(r'(?i)^#\s*ansiblerequires\s+-powershell\s+((Ansible\.ModuleUtils\.[\w\.]+)|' + r'(ansible_collections\.\w+\.\w+\.plugins\.module_utils\.[\w\.]+)|' + r'(\.[\w\.]+))(?P<optional>\s+-Optional){0,1}')), + ] + + self._re_wrapper = re.compile(to_bytes(r'(?i)^#\s*ansiblerequires\s+-wrapper\s+(\w*)')) + self._re_ps_version = re.compile(to_bytes(r'(?i)^#requires\s+\-version\s+([0-9]+(\.[0-9]+){0,3})$')) + self._re_os_version = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-osversion\s+([0-9]+(\.[0-9]+){0,3})$')) + self._re_become = re.compile(to_bytes(r'(?i)^#ansiblerequires\s+\-become$')) + + def scan_module(self, module_data, fqn=None, wrapper=False, powershell=True): + lines = module_data.split(b'\n') + module_utils = set() + if wrapper: + cs_utils = self.cs_utils_wrapper + else: + cs_utils = self.cs_utils_module + + if powershell: + checks = [ + # PS module contains '#Requires -Module Ansible.ModuleUtils.*' + # PS module contains '#AnsibleRequires -Powershell Ansible.*' (or collections module_utils ref) + (self._re_ps_module, self.ps_modules, ".psm1"), + # PS module contains '#AnsibleRequires -CSharpUtil Ansible.*' (or collections module_utils ref) + (self._re_cs_in_ps_module, cs_utils, ".cs"), + ] + else: + checks = [ + # CS module contains 'using Ansible.*;' or 'using ansible_collections.ns.coll.plugins.module_utils.*;' + (self._re_cs_module, cs_utils, ".cs"), + ] + + for line in lines: + for check in checks: + for pattern in check[0]: + match = pattern.match(line) + if match: + # tolerate windows line endings by stripping any remaining + # newline chars + module_util_name = to_text(match.group(1).rstrip()) + match_dict = match.groupdict() + optional = match_dict.get('optional', None) is not None + + if module_util_name not in check[1].keys(): + module_utils.add((module_util_name, check[2], fqn, optional)) + + break + + if powershell: + ps_version_match = self._re_ps_version.match(line) + if ps_version_match: + self._parse_version_match(ps_version_match, "ps_version") + + os_version_match = self._re_os_version.match(line) + if os_version_match: + self._parse_version_match(os_version_match, "os_version") + + # once become is set, no need to keep on checking recursively + if not self.become: + become_match = self._re_become.match(line) + if become_match: + self.become = True + + if wrapper: + wrapper_match = self._re_wrapper.match(line) + if wrapper_match: + self.scan_exec_script(wrapper_match.group(1).rstrip()) + + # recursively drill into each Requires to see if there are any more + # requirements + for m in set(module_utils): + self._add_module(*m, wrapper=wrapper) + + def scan_exec_script(self, name): + # scans lib/ansible/executor/powershell for scripts used in the module + # exec side. It also scans these scripts for any dependencies + name = to_text(name) + if name in self.exec_scripts.keys(): + return + + data = pkgutil.get_data("ansible.executor.powershell", to_native(name + ".ps1")) + if data is None: + raise AnsibleError("Could not find executor powershell script " + "for '%s'" % name) + + b_data = to_bytes(data) + + # remove comments to reduce the payload size in the exec wrappers + if C.DEFAULT_DEBUG: + exec_script = b_data + else: + exec_script = _strip_comments(b_data) + self.exec_scripts[name] = to_bytes(exec_script) + self.scan_module(b_data, wrapper=True, powershell=True) + + def _add_module(self, name, ext, fqn, optional, wrapper=False): + m = to_text(name) + + util_fqn = None + + if m.startswith("Ansible."): + # Builtin util, use plugin loader to get the data + mu_path = ps_module_utils_loader.find_plugin(m, ext) + + if not mu_path: + if optional: + return + + raise AnsibleError('Could not find imported module support code ' + 'for \'%s\'' % m) + + module_util_data = to_bytes(_slurp(mu_path)) + else: + # Collection util, load the package data based on the util import. + + submodules = m.split(".") + if m.startswith('.'): + fqn_submodules = fqn.split('.') + for submodule in submodules: + if submodule: + break + del fqn_submodules[-1] + + submodules = fqn_submodules + [s for s in submodules if s] + + n_package_name = to_native('.'.join(submodules[:-1]), errors='surrogate_or_strict') + n_resource_name = to_native(submodules[-1] + ext, errors='surrogate_or_strict') + + try: + module_util = import_module(n_package_name) + pkg_data = pkgutil.get_data(n_package_name, n_resource_name) + if pkg_data is None: + raise ImportError("No package data found") + + module_util_data = to_bytes(pkg_data, errors='surrogate_or_strict') + util_fqn = to_text("%s.%s " % (n_package_name, submodules[-1]), errors='surrogate_or_strict') + + # Get the path of the util which is required for coverage collection. + resource_paths = list(module_util.__path__) + if len(resource_paths) != 1: + # This should never happen with a collection but we are just being defensive about it. + raise AnsibleError("Internal error: Referenced module_util package '%s' contains 0 or multiple " + "import locations when we only expect 1." % n_package_name) + mu_path = os.path.join(resource_paths[0], n_resource_name) + except (ImportError, OSError) as err: + if getattr(err, "errno", errno.ENOENT) == errno.ENOENT: + if optional: + return + + raise AnsibleError('Could not find collection imported module support code for \'%s\'' + % to_native(m)) + + else: + raise + + util_info = { + 'data': module_util_data, + 'path': to_text(mu_path), + } + if ext == ".psm1": + self.ps_modules[m] = util_info + else: + if wrapper: + self.cs_utils_wrapper[m] = util_info + else: + self.cs_utils_module[m] = util_info + self.scan_module(module_util_data, fqn=util_fqn, wrapper=wrapper, powershell=(ext == ".psm1")) + + def _parse_version_match(self, match, attribute): + new_version = to_text(match.group(1)).rstrip() + + # PowerShell cannot cast a string of "1" to Version, it must have at + # least the major.minor for it to be valid so we append 0 + if match.group(2) is None: + new_version = "%s.0" % new_version + + existing_version = getattr(self, attribute, None) + if existing_version is None: + setattr(self, attribute, new_version) + else: + # determine which is the latest version and set that + if LooseVersion(new_version) > LooseVersion(existing_version): + setattr(self, attribute, new_version) + + +def _slurp(path): + if not os.path.exists(path): + raise AnsibleError("imported module support code does not exist at %s" + % os.path.abspath(path)) + fd = open(path, 'rb') + data = fd.read() + fd.close() + return data + + +def _strip_comments(source): + # Strip comments and blank lines from the wrapper + buf = [] + start_block = False + for line in source.splitlines(): + l = line.strip() + + if start_block and l.endswith(b'#>'): + start_block = False + continue + elif start_block: + continue + elif l.startswith(b'<#'): + start_block = True + continue + elif not l or l.startswith(b'#'): + continue + + buf.append(line) + return b'\n'.join(buf) + + +def _create_powershell_wrapper(b_module_data, module_path, module_args, + environment, async_timeout, become, + become_method, become_user, become_password, + become_flags, substyle, task_vars, module_fqn): + # creates the manifest/wrapper used in PowerShell/C# modules to enable + # things like become and async - this is also called in action/script.py + + # FUTURE: add process_wrapper.ps1 to run module_wrapper in a new process + # if running under a persistent connection and substyle is C# so we + # don't have type conflicts + finder = PSModuleDepFinder() + if substyle != 'script': + # don't scan the module for util dependencies and other Ansible related + # flags if the substyle is 'script' which is set by action/script + finder.scan_module(b_module_data, fqn=module_fqn, powershell=(substyle == "powershell")) + + module_wrapper = "module_%s_wrapper" % substyle + exec_manifest = dict( + module_entry=to_text(base64.b64encode(b_module_data)), + powershell_modules=dict(), + csharp_utils=dict(), + csharp_utils_module=list(), # csharp_utils only required by a module + module_args=module_args, + actions=[module_wrapper], + environment=environment, + encoded_output=False, + ) + finder.scan_exec_script(module_wrapper) + + if async_timeout > 0: + finder.scan_exec_script('exec_wrapper') + finder.scan_exec_script('async_watchdog') + finder.scan_exec_script('async_wrapper') + + exec_manifest["actions"].insert(0, 'async_watchdog') + exec_manifest["actions"].insert(0, 'async_wrapper') + exec_manifest["async_jid"] = str(random.randint(0, 999999999999)) + exec_manifest["async_timeout_sec"] = async_timeout + exec_manifest["async_startup_timeout"] = C.config.get_config_value("WIN_ASYNC_STARTUP_TIMEOUT", variables=task_vars) + + if become and resource_from_fqcr(become_method) == 'runas': # runas and namespace.collection.runas + finder.scan_exec_script('exec_wrapper') + finder.scan_exec_script('become_wrapper') + + exec_manifest["actions"].insert(0, 'become_wrapper') + exec_manifest["become_user"] = become_user + exec_manifest["become_password"] = become_password + exec_manifest['become_flags'] = become_flags + + exec_manifest['min_ps_version'] = finder.ps_version + exec_manifest['min_os_version'] = finder.os_version + if finder.become and 'become_wrapper' not in exec_manifest['actions']: + finder.scan_exec_script('exec_wrapper') + finder.scan_exec_script('become_wrapper') + + exec_manifest['actions'].insert(0, 'become_wrapper') + exec_manifest['become_user'] = 'SYSTEM' + exec_manifest['become_password'] = None + exec_manifest['become_flags'] = None + + coverage_manifest = dict( + module_path=module_path, + module_util_paths=dict(), + output=None, + ) + coverage_output = C.config.get_config_value('COVERAGE_REMOTE_OUTPUT', variables=task_vars) + if coverage_output and substyle == 'powershell': + finder.scan_exec_script('coverage_wrapper') + coverage_manifest['output'] = coverage_output + + coverage_enabled = C.config.get_config_value('COVERAGE_REMOTE_PATHS', variables=task_vars) + coverage_manifest['path_filter'] = coverage_enabled + + # make sure Ansible.ModuleUtils.AddType is added if any C# utils are used + if len(finder.cs_utils_wrapper) > 0 or len(finder.cs_utils_module) > 0: + finder._add_module(b"Ansible.ModuleUtils.AddType", ".psm1", None, False, + wrapper=False) + + # exec_wrapper is only required to be part of the payload if using + # become or async, to save on payload space we check if exec_wrapper has + # already been added, and remove it manually if it hasn't later + exec_required = "exec_wrapper" in finder.exec_scripts.keys() + finder.scan_exec_script("exec_wrapper") + # must contain an empty newline so it runs the begin/process/end block + finder.exec_scripts["exec_wrapper"] += b"\n\n" + + exec_wrapper = finder.exec_scripts["exec_wrapper"] + if not exec_required: + finder.exec_scripts.pop("exec_wrapper") + + for name, data in finder.exec_scripts.items(): + b64_data = to_text(base64.b64encode(data)) + exec_manifest[name] = b64_data + + for name, data in finder.ps_modules.items(): + b64_data = to_text(base64.b64encode(data['data'])) + exec_manifest['powershell_modules'][name] = b64_data + coverage_manifest['module_util_paths'][name] = data['path'] + + cs_utils = {} + for cs_util in [finder.cs_utils_wrapper, finder.cs_utils_module]: + for name, data in cs_util.items(): + cs_utils[name] = data['data'] + + for name, data in cs_utils.items(): + b64_data = to_text(base64.b64encode(data)) + exec_manifest['csharp_utils'][name] = b64_data + exec_manifest['csharp_utils_module'] = list(finder.cs_utils_module.keys()) + + # To save on the data we are sending across we only add the coverage info if coverage is being run + if 'coverage_wrapper' in exec_manifest: + exec_manifest['coverage'] = coverage_manifest + + b_json = to_bytes(json.dumps(exec_manifest)) + # delimit the payload JSON from the wrapper to keep sensitive contents out of scriptblocks (which can be logged) + b_data = exec_wrapper + b'\0\0\0\0' + b_json + return b_data diff --git a/lib/ansible/executor/powershell/module_powershell_wrapper.ps1 b/lib/ansible/executor/powershell/module_powershell_wrapper.ps1 new file mode 100644 index 0000000..c35c84c --- /dev/null +++ b/lib/ansible/executor/powershell/module_powershell_wrapper.ps1 @@ -0,0 +1,75 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +#AnsibleRequires -Wrapper module_wrapper + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting module_powershell_wrapper" "module_powershell_wrapper" + +$module_name = $Payload.module_args["_ansible_module_name"] +Write-AnsibleLog "INFO - building module payload for '$module_name'" "module_powershell_wrapper" + +# compile any C# module utils passed in from the controller, Add-CSharpType is +# automatically added to the payload manifest if any csharp util is set +$csharp_utils = [System.Collections.ArrayList]@() +foreach ($csharp_util in $Payload.csharp_utils_module) { + Write-AnsibleLog "INFO - adding $csharp_util to list of C# references to compile" "module_powershell_wrapper" + $util_code = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.csharp_utils[$csharp_util])) + $csharp_utils.Add($util_code) > $null +} +if ($csharp_utils.Count -gt 0) { + $add_type_b64 = $Payload.powershell_modules["Ansible.ModuleUtils.AddType"] + $add_type = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($add_type_b64)) + New-Module -Name Ansible.ModuleUtils.AddType -ScriptBlock ([ScriptBlock]::Create($add_type)) | Import-Module > $null + + # add any C# references so the module does not have to do so + $new_tmp = [System.Environment]::ExpandEnvironmentVariables($Payload.module_args["_ansible_remote_tmp"]) + Add-CSharpType -References $csharp_utils -TempPath $new_tmp -IncludeDebugInfo +} + +if ($Payload.ContainsKey("coverage") -and $null -ne $host.Runspace -and $null -ne $host.Runspace.Debugger) { + $entrypoint = $payload.coverage_wrapper + + $params = @{ + Payload = $Payload + } +} +else { + # get the common module_wrapper code and invoke that to run the module + $module = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.module_entry)) + $variables = [System.Collections.ArrayList]@(@{ Name = "complex_args"; Value = $Payload.module_args; Scope = "Global" }) + $entrypoint = $Payload.module_wrapper + + $params = @{ + Scripts = @($script:common_functions, $module) + Variables = $variables + Environment = $Payload.environment + Modules = $Payload.powershell_modules + ModuleName = $module_name + } +} + +$entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($entrypoint)) +$entrypoint = [ScriptBlock]::Create($entrypoint) + +try { + &$entrypoint @params +} +catch { + # failed to invoke the PowerShell module, capture the exception and + # output a pretty error for Ansible to parse + $result = @{ + msg = "Failed to invoke PowerShell module: $($_.Exception.Message)" + failed = $true + exception = (Format-AnsibleException -ErrorRecord $_) + } + Write-Output -InputObject (ConvertTo-Json -InputObject $result -Depth 99 -Compress) + $host.SetShouldExit(1) +} + +Write-AnsibleLog "INFO - ending module_powershell_wrapper" "module_powershell_wrapper" diff --git a/lib/ansible/executor/powershell/module_script_wrapper.ps1 b/lib/ansible/executor/powershell/module_script_wrapper.ps1 new file mode 100644 index 0000000..dd8420f --- /dev/null +++ b/lib/ansible/executor/powershell/module_script_wrapper.ps1 @@ -0,0 +1,22 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +param( + [Parameter(Mandatory = $true)][System.Collections.IDictionary]$Payload +) + +#AnsibleRequires -Wrapper module_wrapper + +$ErrorActionPreference = "Stop" + +Write-AnsibleLog "INFO - starting module_script_wrapper" "module_script_wrapper" + +$script = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.module_entry)) + +# get the common module_wrapper code and invoke that to run the module +$entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($payload.module_wrapper)) +$entrypoint = [ScriptBlock]::Create($entrypoint) + +&$entrypoint -Scripts $script -Environment $Payload.environment -ModuleName "script" + +Write-AnsibleLog "INFO - ending module_script_wrapper" "module_script_wrapper" diff --git a/lib/ansible/executor/powershell/module_wrapper.ps1 b/lib/ansible/executor/powershell/module_wrapper.ps1 new file mode 100644 index 0000000..20a9677 --- /dev/null +++ b/lib/ansible/executor/powershell/module_wrapper.ps1 @@ -0,0 +1,226 @@ +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +<# +.SYNOPSIS +Invokes an Ansible module in a new Runspace. This cmdlet will output the +module's output and write any errors to the error stream of the current +host. + +.PARAMETER Scripts +[Object[]] String or ScriptBlocks to execute. + +.PARAMETER Variables +[System.Collections.ArrayList] The variables to set in the new Pipeline. +Each value is a hashtable that contains the parameters to use with +Set-Variable; + Name: the name of the variable to set + Value: the value of the variable to set + Scope: the scope of the variable + +.PARAMETER Environment +[System.Collections.IDictionary] A Dictionary of environment key/values to +set in the new Pipeline. + +.PARAMETER Modules +[System.Collections.IDictionary] A Dictionary of PowerShell modules to +import into the new Pipeline. The key is the name of the module and the +value is a base64 string of the module util code. + +.PARAMETER ModuleName +[String] The name of the module that is being executed. + +.PARAMETER Breakpoints +A list of line breakpoints to add to the runspace debugger. This is used to +track module and module_utils coverage. +#> +param( + [Object[]]$Scripts, + [System.Collections.ArrayList][AllowEmptyCollection()]$Variables, + [System.Collections.IDictionary]$Environment, + [System.Collections.IDictionary]$Modules, + [String]$ModuleName, + [System.Management.Automation.LineBreakpoint[]]$Breakpoints = @() +) + +Write-AnsibleLog "INFO - creating new PowerShell pipeline for $ModuleName" "module_wrapper" +$ps = [PowerShell]::Create() + +# do not set ErrorActionPreference for script +if ($ModuleName -ne "script") { + $ps.Runspace.SessionStateProxy.SetVariable("ErrorActionPreference", "Stop") +} + +# force input encoding to preamble-free UTF8 so PS sub-processes (eg, +# Start-Job) don't blow up. This is only required for WinRM, a PSRP +# runspace doesn't have a host console and this will bomb out +if ($host.Name -eq "ConsoleHost") { + Write-AnsibleLog "INFO - setting console input encoding to UTF8 for $ModuleName" "module_wrapper" + $ps.AddScript('[Console]::InputEncoding = New-Object Text.UTF8Encoding $false').AddStatement() > $null +} + +# set the variables +foreach ($variable in $Variables) { + Write-AnsibleLog "INFO - setting variable '$($variable.Name)' for $ModuleName" "module_wrapper" + $ps.AddCommand("Set-Variable").AddParameters($variable).AddStatement() > $null +} + +# set the environment vars +if ($Environment) { + # Escaping quotes can be problematic, instead just pass the string to the runspace and set it directly. + Write-AnsibleLog "INFO - setting environment vars for $ModuleName" "module_wrapper" + $ps.Runspace.SessionStateProxy.SetVariable("_AnsibleEnvironment", $Environment) + $ps.AddScript(@' +foreach ($env_kv in $_AnsibleEnvironment.GetEnumerator()) { + [System.Environment]::SetEnvironmentVariable($env_kv.Key, $env_kv.Value) +} +'@).AddStatement() > $null +} + +# import the PS modules +if ($Modules) { + foreach ($module in $Modules.GetEnumerator()) { + Write-AnsibleLog "INFO - create module util '$($module.Key)' for $ModuleName" "module_wrapper" + $module_name = $module.Key + $module_code = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($module.Value)) + $ps.AddCommand("New-Module").AddParameters(@{Name = $module_name; ScriptBlock = [ScriptBlock]::Create($module_code) }) > $null + $ps.AddCommand("Import-Module").AddParameter("WarningAction", "SilentlyContinue") > $null + $ps.AddCommand("Out-Null").AddStatement() > $null + } +} + +# redefine Write-Host to dump to output instead of failing +# lots of scripts still use it +$ps.AddScript('Function Write-Host($msg) { Write-Output -InputObject $msg }').AddStatement() > $null + +# add the scripts and run +foreach ($script in $Scripts) { + $ps.AddScript($script).AddStatement() > $null +} + +if ($Breakpoints.Count -gt 0) { + Write-AnsibleLog "INFO - adding breakpoint to runspace that will run the modules" "module_wrapper" + if ($PSVersionTable.PSVersion.Major -eq 3) { + # The SetBreakpoints method was only added in PowerShell v4+. We need to rely on a private method to + # achieve the same functionality in this older PowerShell version. This should be removed once we drop + # support for PowerShell v3. + $set_method = $ps.Runspace.Debugger.GetType().GetMethod( + 'AddLineBreakpoint', [System.Reflection.BindingFlags]'Instance, NonPublic' + ) + foreach ($b in $Breakpoints) { + $set_method.Invoke($ps.Runspace.Debugger, [Object[]]@(, $b)) > $null + } + } + else { + $ps.Runspace.Debugger.SetBreakpoints($Breakpoints) + } +} + +Write-AnsibleLog "INFO - start module exec with Invoke() - $ModuleName" "module_wrapper" + +# temporarily override the stdout stream and create our own in a StringBuilder +# we use this to ensure there's always an Out pipe and that we capture the +# output for things like async or psrp +$orig_out = [System.Console]::Out +$sb = New-Object -TypeName System.Text.StringBuilder +$new_out = New-Object -TypeName System.IO.StringWriter -ArgumentList $sb +try { + [System.Console]::SetOut($new_out) + $module_output = $ps.Invoke() +} +catch { + # uncaught exception while executing module, present a prettier error for + # Ansible to parse + $error_params = @{ + Message = "Unhandled exception while executing module" + ErrorRecord = $_ + } + + # Be more defensive when trying to find the InnerException in case it isn't + # set. This shouldn't ever be the case but if it is then it makes it more + # difficult to track down the problem. + if ($_.Exception.PSObject.Properties.Name -contains "InnerException") { + $inner_exception = $_.Exception.InnerException + if ($inner_exception.PSObject.Properties.Name -contains "ErrorRecord") { + $error_params.ErrorRecord = $inner_exception.ErrorRecord + } + } + + Write-AnsibleError @error_params + $host.SetShouldExit(1) + return +} +finally { + [System.Console]::SetOut($orig_out) + $new_out.Dispose() +} + +# other types of errors may not throw an exception in Invoke but rather just +# set the pipeline state to failed +if ($ps.InvocationStateInfo.State -eq "Failed" -and $ModuleName -ne "script") { + $reason = $ps.InvocationStateInfo.Reason + $error_params = @{ + Message = "Unhandled exception while executing module" + } + + # The error record should always be set on the reason but this does not + # always happen on Server 2008 R2 for some reason (probably memory hotfix). + # Be defensive when trying to get the error record and fall back to other + # options. + if ($null -eq $reason) { + $error_params.Message += ": Unknown error" + } + elseif ($reason.PSObject.Properties.Name -contains "ErrorRecord") { + $error_params.ErrorRecord = $reason.ErrorRecord + } + else { + $error_params.Message += ": $($reason.ToString())" + } + + Write-AnsibleError @error_params + $host.SetShouldExit(1) + return +} + +Write-AnsibleLog "INFO - module exec ended $ModuleName" "module_wrapper" +$stdout = $sb.ToString() +if ($stdout) { + Write-Output -InputObject $stdout +} +if ($module_output.Count -gt 0) { + # do not output if empty collection + Write-AnsibleLog "INFO - using the output stream for module output - $ModuleName" "module_wrapper" + Write-Output -InputObject ($module_output -join "`r`n") +} + +# we attempt to get the return code from the LASTEXITCODE variable +# this is set explicitly in newer style variables when calling +# ExitJson and FailJson. If set we set the current hosts' exit code +# to that same value +$rc = $ps.Runspace.SessionStateProxy.GetVariable("LASTEXITCODE") +if ($null -ne $rc) { + Write-AnsibleLog "INFO - got an rc of $rc from $ModuleName exec" "module_wrapper" + $host.SetShouldExit($rc) +} + +# PS3 doesn't properly set HadErrors in many cases, inspect the error stream as a fallback +# with the trap handler that's now in place, this should only write to the output if +# $ErrorActionPreference != "Stop", that's ok because this is sent to the stderr output +# for a user to manually debug if something went horribly wrong +if ($ps.HadErrors -or ($PSVersionTable.PSVersion.Major -lt 4 -and $ps.Streams.Error.Count -gt 0)) { + Write-AnsibleLog "WARN - module had errors, outputting error info $ModuleName" "module_wrapper" + # if the rc wasn't explicitly set, we return an exit code of 1 + if ($null -eq $rc) { + $host.SetShouldExit(1) + } + + # output each error to the error stream of the current pipeline + foreach ($err in $ps.Streams.Error) { + $error_msg = Format-AnsibleException -ErrorRecord $err + + # need to use the current hosts's UI class as we may not have + # a console to write the stderr to, e.g. psrp + Write-AnsibleLog "WARN - error msg for for $($ModuleName):`r`n$error_msg" "module_wrapper" + $host.UI.WriteErrorLine($error_msg) + } +} |