diff options
Diffstat (limited to 'qa/workunits/windows/libvirt_vm')
| -rw-r--r-- | qa/workunits/windows/libvirt_vm/autounattend.xml | 157 | ||||
| -rw-r--r-- | qa/workunits/windows/libvirt_vm/first-logon.ps1 | 42 | ||||
| -rw-r--r-- | qa/workunits/windows/libvirt_vm/setup.ps1 | 43 | ||||
| -rwxr-xr-x | qa/workunits/windows/libvirt_vm/setup.sh | 162 | ||||
| -rw-r--r-- | qa/workunits/windows/libvirt_vm/utils.ps1 | 130 |
5 files changed, 534 insertions, 0 deletions
diff --git a/qa/workunits/windows/libvirt_vm/autounattend.xml b/qa/workunits/windows/libvirt_vm/autounattend.xml new file mode 100644 index 000000000..c3cdf3171 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/autounattend.xml @@ -0,0 +1,157 @@ +<?xml version="1.0" encoding="utf-8"?> +<unattend xmlns="urn:schemas-microsoft-com:unattend"> + + <settings pass="windowsPE"> + + <component name="Microsoft-Windows-International-Core-WinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <SetupUILanguage> + <UILanguage>en-US</UILanguage> + </SetupUILanguage> + <SystemLocale>en-US</SystemLocale> + <UILanguage>en-US</UILanguage> + <UserLocale>en-US</UserLocale> + </component> + + <component name="Microsoft-Windows-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + <DiskConfiguration> + <WillShowUI>OnError</WillShowUI> + <Disk wcm:action="add"> + <CreatePartitions> + <CreatePartition wcm:action="add"> + <Order>1</Order> + <Size>100</Size> + <Type>Primary</Type> + </CreatePartition> + <CreatePartition wcm:action="add"> + <Order>2</Order> + <Extend>true</Extend> + <Type>Primary</Type> + </CreatePartition> + </CreatePartitions> + <ModifyPartitions> + <ModifyPartition wcm:action="add"> + <Active>true</Active> + <Label>Boot</Label> + <Format>NTFS</Format> + <Order>1</Order> + <PartitionID>1</PartitionID> + </ModifyPartition> + <ModifyPartition wcm:action="add"> + <Format>NTFS</Format> + <Order>2</Order> + <PartitionID>2</PartitionID> + <Label>System</Label> + </ModifyPartition> + </ModifyPartitions> + <DiskID>0</DiskID> + <WillWipeDisk>true</WillWipeDisk> + </Disk> + </DiskConfiguration> + + <ImageInstall> + <OSImage> + <InstallTo> + <PartitionID>2</PartitionID> + <DiskID>0</DiskID> + </InstallTo> + <InstallToAvailablePartition>false</InstallToAvailablePartition> + <WillShowUI>OnError</WillShowUI> + <InstallFrom> + <MetaData wcm:action="add"> + <Key>/IMAGE/NAME</Key> + <Value>Windows Server 2019 SERVERSTANDARDCORE</Value> + </MetaData> + </InstallFrom> + </OSImage> + </ImageInstall> + + <UserData> + <!-- Product Key from http://technet.microsoft.com/en-us/library/jj612867.aspx --> + <ProductKey> + <!-- Do not uncomment the Key element if you are using trial ISOs --> + <!-- You must uncomment the Key element (and optionally insert your own key) if you are using retail or volume license ISOs --> + <!-- <Key></Key> --> + <WillShowUI>OnError</WillShowUI> + </ProductKey> + <AcceptEula>true</AcceptEula> + </UserData> + + </component> + + <component name="Microsoft-Windows-PnpCustomizationsWinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <DriverPaths> + <PathAndCredentials wcm:action="add" wcm:keyValue="1"> + <Path>E:\NetKVM\2k19\amd64\</Path> + </PathAndCredentials> + <PathAndCredentials wcm:action="add" wcm:keyValue="2"> + <Path>E:\viostor\2k19\amd64\</Path> + </PathAndCredentials> + <PathAndCredentials wcm:action="add" wcm:keyValue="3"> + <Path>E:\vioserial\2k19\amd64\</Path> + </PathAndCredentials> + </DriverPaths> + </component> + + </settings> + + <settings pass="oobeSystem"> + <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + <VisualEffects> + <FontSmoothing>ClearType</FontSmoothing> + </VisualEffects> + + <UserAccounts> + <!-- + Password to be used only during initial provisioning. + Must be reset with final Sysprep. + --> + <AdministratorPassword> + <Value>Passw0rd</Value> + <PlainText>true</PlainText> + </AdministratorPassword> + </UserAccounts> + + <AutoLogon> + <Password> + <Value>Passw0rd</Value> + <PlainText>true</PlainText> + </Password> + <Enabled>true</Enabled> + <Username>Administrator</Username> + </AutoLogon> + + <ComputerName>*</ComputerName> + + <OOBE> + <NetworkLocation>Work</NetworkLocation> + <HideEULAPage>true</HideEULAPage> + <ProtectYourPC>3</ProtectYourPC> + <SkipMachineOOBE>true</SkipMachineOOBE> + <SkipUserOOBE>true</SkipUserOOBE> + </OOBE> + + <FirstLogonCommands> + + <SynchronousCommand wcm:action="add"> + <CommandLine>%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell -NoLogo -NonInteractive -ExecutionPolicy RemoteSigned -File A:\first-logon.ps1</CommandLine> + <Order>1</Order> + </SynchronousCommand> + + </FirstLogonCommands> + + </component> + + </settings> + + <settings pass="specialize"> + + <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <TimeZone>UTC</TimeZone> + <ComputerName>*</ComputerName> + </component> + + </settings> + +</unattend> diff --git a/qa/workunits/windows/libvirt_vm/first-logon.ps1 b/qa/workunits/windows/libvirt_vm/first-logon.ps1 new file mode 100644 index 000000000..654b836bb --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/first-logon.ps1 @@ -0,0 +1,42 @@ +$ErrorActionPreference = "Stop" + +. "${PSScriptRoot}\utils.ps1" + +$VIRTIO_WIN_PATH = "E:\" + +# Install QEMU quest agent +Write-Output "Installing QEMU guest agent" +$p = Start-Process -FilePath "msiexec.exe" -ArgumentList @("/i", "${VIRTIO_WIN_PATH}\guest-agent\qemu-ga-x86_64.msi", "/qn") -NoNewWindow -PassThru -Wait +if($p.ExitCode) { + Throw "The QEMU guest agent installation failed. Exit code: $($p.ExitCode)" +} +Write-Output "Successfully installed QEMU guest agent" + +# Install OpenSSH server +Start-ExecuteWithRetry { + Get-WindowsCapability -Online -Name OpenSSH* | Add-WindowsCapability -Online +} + +# Start OpenSSH server +Set-Service -Name "sshd" -StartupType Automatic +Start-Service -Name "sshd" + +# Set PowerShell as default SSH shell +New-ItemProperty -PropertyType String -Force -Name DefaultShell -Path "HKLM:\SOFTWARE\OpenSSH" -Value (Get-Command powershell.exe).Source + +# Create SSH firewall rule +New-NetFirewallRule -Name "sshd" -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + +# Authorize the SSH key +$authorizedKeysFile = Join-Path $env:ProgramData "ssh\administrators_authorized_keys" +Set-Content -Path $authorizedKeysFile -Value (Get-Content "${PSScriptRoot}\id_rsa.pub") -Encoding ascii +$acl = Get-Acl $authorizedKeysFile +$acl.SetAccessRuleProtection($true, $false) +$administratorsRule = New-Object system.security.accesscontrol.filesystemaccessrule("Administrators", "FullControl", "Allow") +$systemRule = New-Object system.security.accesscontrol.filesystemaccessrule("SYSTEM", "FullControl", "Allow") +$acl.SetAccessRule($administratorsRule) +$acl.SetAccessRule($systemRule) +$acl | Set-Acl + +# Reboot the machine to complete first logon process +Restart-Computer -Force -Confirm:$false diff --git a/qa/workunits/windows/libvirt_vm/setup.ps1 b/qa/workunits/windows/libvirt_vm/setup.ps1 new file mode 100644 index 000000000..550fb274e --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/setup.ps1 @@ -0,0 +1,43 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$PYTHON3_URL = "https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe" +$FIO_URL = "https://bsdio.com/fio/releases/fio-3.27-x64.msi" +$VC_REDIST_URL = "https://aka.ms/vs/17/release/vc_redist.x64.exe" + +. "${PSScriptRoot}\utils.ps1" + +function Install-VCRedist { + Write-Output "Installing Visual Studio Redistributable x64" + Install-Tool -URL $VC_REDIST_URL -Params @("/quiet", "/norestart") + Write-Output "Successfully installed Visual Studio Redistributable x64" +} + +function Install-Python3 { + Write-Output "Installing Python3" + Install-Tool -URL $PYTHON3_URL -Params @("/quiet", "InstallAllUsers=1", "PrependPath=1") + Add-ToPathEnvVar -Path @("${env:ProgramFiles}\Python310\", "${env:ProgramFiles}\Python310\Scripts\") + Write-Output "Installing pip dependencies" + Start-ExecuteWithRetry { + Invoke-CommandLine "pip3.exe" "install prettytable" + } + Write-Output "Successfully installed Python3" +} + +function Install-FIO { + Write-Output "Installing FIO" + Install-Tool -URL $FIO_URL -Params @("/qn", "/l*v", "$env:TEMP\fio-install.log", "/norestart") + Write-Output "Successfully installed FIO" +} + +Install-VCRedist +Install-Python3 +Install-FIO + +# Pre-append WNBD and Ceph to PATH +Add-ToPathEnvVar -Path @( + "${env:SystemDrive}\wnbd\binaries", + "${env:SystemDrive}\ceph") + +# This will refresh the PATH for new SSH sessions +Restart-Service -Force -Name "sshd" diff --git a/qa/workunits/windows/libvirt_vm/setup.sh b/qa/workunits/windows/libvirt_vm/setup.sh new file mode 100755 index 000000000..51e91ec42 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/setup.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -ex + +WINDOWS_SERVER_2019_ISO_URL=${WINDOWS_SERVER_2019_ISO_URL:-"https://software-download.microsoft.com/download/pr/17763.737.190906-2324.rs5_release_svc_refresh_SERVER_EVAL_x64FRE_en-us_1.iso"} +VIRTIO_WIN_ISO_URL=${VIRTIO_WIN_ISO_URL:-"https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso"} + +DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" + +# Use build_utils.sh from ceph-build +curl --retry-max-time 30 --retry 10 -L -o ${DIR}/build_utils.sh https://raw.githubusercontent.com/ceph/ceph-build/main/scripts/build_utils.sh +source ${DIR}/build_utils.sh + +# Helper function to restart the Windows VM +function restart_windows_vm() { + echo "Restarting Windows VM" + ssh_exec "cmd.exe /c 'shutdown.exe /r /t 0 & sc.exe stop sshd'" + SECONDS=0 + TIMEOUT=${1:-600} + while true; do + if [[ $SECONDS -gt $TIMEOUT ]]; then + echo "Timeout waiting for the VM to start" + exit 1 + fi + ssh_exec hostname || { + echo "Cannot execute SSH commands yet" + sleep 10 + continue + } + break + done + echo "Windows VM restarted" +} + +# Install libvirt with KVM +retrycmd_if_failure 5 0 5m sudo apt-get update +retrycmd_if_failure 5 0 10m sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients virtinst + +# Download ISO images +echo "Downloading virtio-win ISO" +retrycmd_if_failure 5 0 30m curl -C - -L $VIRTIO_WIN_ISO_URL -o ${DIR}/virtio-win.iso +echo "Downloading Windows Server 2019 ISO" +retrycmd_if_failure 5 0 60m curl -C - -L $WINDOWS_SERVER_2019_ISO_URL -o ${DIR}/windows-server-2019.iso + +# Create virtual floppy image with the unattended instructions to install Windows Server 2019 +echo "Creating floppy image" +qemu-img create -f raw ${DIR}/floppy.img 1440k +mkfs.msdos -s 1 ${DIR}/floppy.img +mkdir ${DIR}/floppy +sudo mount ${DIR}/floppy.img ${DIR}/floppy +ssh-keygen -b 2048 -t rsa -f ${DIR}/id_rsa -q -N "" +sudo cp \ + ${DIR}/autounattend.xml \ + ${DIR}/first-logon.ps1 \ + ${DIR}/id_rsa.pub \ + ${DIR}/utils.ps1 \ + ${DIR}/setup.ps1 \ + ${DIR}/floppy/ +sudo umount ${DIR}/floppy +rmdir ${DIR}/floppy + +echo "Starting libvirt VM" +qemu-img create -f qcow2 ${DIR}/ceph-win-ltsc2019.qcow2 50G +VM_NAME="ceph-win-ltsc2019" +sudo virt-install \ + --name $VM_NAME \ + --os-variant win2k19 \ + --boot hd,cdrom \ + --virt-type kvm \ + --graphics spice \ + --cpu host \ + --vcpus 4 \ + --memory 4096 \ + --disk ${DIR}/floppy.img,device=floppy \ + --disk ${DIR}/ceph-win-ltsc2019.qcow2,bus=virtio \ + --disk ${DIR}/windows-server-2019.iso,device=cdrom \ + --disk ${DIR}/virtio-win.iso,device=cdrom \ + --network network=default,model=virtio \ + --controller type=virtio-serial \ + --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \ + --noautoconsol + +export SSH_USER="administrator" +export SSH_KNOWN_HOSTS_FILE="${DIR}/known_hosts" +export SSH_KEY="${DIR}/id_rsa" + +SECONDS=0 +TIMEOUT=1800 +SLEEP_SECS=30 +while true; do + if [[ $SECONDS -gt $TIMEOUT ]]; then + echo "Timeout waiting for the VM to start" + exit 1 + fi + VM_IP=$(sudo virsh domifaddr --source agent --interface Ethernet --full $VM_NAME | grep ipv4 | awk '{print $4}' | cut -d '/' -f1) || { + echo "Retrying in $SLEEP_SECS seconds" + sleep $SLEEP_SECS + continue + } + ssh-keyscan -H $VM_IP &> $SSH_KNOWN_HOSTS_FILE || { + echo "SSH is not reachable yet" + sleep $SLEEP_SECS + continue + } + SSH_ADDRESS=$VM_IP ssh_exec hostname || { + echo "Cannot execute SSH commands yet" + sleep $SLEEP_SECS + continue + } + break +done +export SSH_ADDRESS=$VM_IP + +scp_upload ${DIR}/utils.ps1 /utils.ps1 +scp_upload ${DIR}/setup.ps1 /setup.ps1 +SSH_TIMEOUT=1h ssh_exec /setup.ps1 + +cd $DIR + +# Get the helper script to download Chacra builds +retrycmd_if_failure 10 5 1m curl -L -o ./get-chacra-bin.py https://raw.githubusercontent.com/ceph/ceph-win32-tests/main/get-bin.py +chmod +x ./get-chacra-bin.py + +# Download latest WNBD build from Chacra +retrycmd_if_failure 10 0 10m ./get-chacra-bin.py --project wnbd --filename wnbd.zip +scp_upload wnbd.zip /wnbd.zip +ssh_exec tar.exe xzvf /wnbd.zip -C / + +# Install WNBD driver +ssh_exec Import-Certificate -FilePath /wnbd/driver/wnbd.cer -Cert Cert:\\LocalMachine\\Root +ssh_exec Import-Certificate -FilePath /wnbd/driver/wnbd.cer -Cert Cert:\\LocalMachine\\TrustedPublisher +ssh_exec /wnbd/binaries/wnbd-client.exe install-driver /wnbd/driver/wnbd.inf +restart_windows_vm +ssh_exec wnbd-client.exe -v + +# Download Ceph Windows build from Chacra +CEPH_REPO_FILE="/etc/apt/sources.list.d/ceph.list" +PROJECT=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $4}') +BRANCH=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $5}') +SHA1=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $6}') +retrycmd_if_failure 10 0 10m ./get-chacra-bin.py --project $PROJECT --branchname $BRANCH --sha1 $SHA1 --filename ceph.zip + +# Install Ceph on Windows +SSH_TIMEOUT=5m scp_upload ./ceph.zip /ceph.zip +SSH_TIMEOUT=10m ssh_exec tar.exe xzvf /ceph.zip -C / +ssh_exec "New-Service -Name ceph-rbd -BinaryPathName 'c:\ceph\rbd-wnbd.exe service'" +ssh_exec Start-Service -Name ceph-rbd +ssh_exec rbd.exe -v + +# Setup Ceph configs and directories +ssh_exec mkdir -force /etc/ceph, /var/run/ceph, /var/log/ceph +for i in $(ls /etc/ceph); do + scp_upload /etc/ceph/$i /etc/ceph/$i +done + +cat << EOF > ${DIR}/connection_info.sh +export SSH_USER="${SSH_USER}" +export SSH_KNOWN_HOSTS_FILE="${SSH_KNOWN_HOSTS_FILE}" +export SSH_KEY="${SSH_KEY}" +export SSH_ADDRESS="${SSH_ADDRESS}" +EOF + +echo "Windows Server 2019 libvirt testing VM is ready" diff --git a/qa/workunits/windows/libvirt_vm/utils.ps1 b/qa/workunits/windows/libvirt_vm/utils.ps1 new file mode 100644 index 000000000..f29ab79f5 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/utils.ps1 @@ -0,0 +1,130 @@ +function Invoke-CommandLine { + Param( + [Parameter(Mandatory=$true)] + [String]$Command, + [String]$Arguments, + [Int[]]$AllowedExitCodes=@(0) + ) + & $Command $Arguments.Split(" ") + if($LASTEXITCODE -notin $AllowedExitCodes) { + Throw "$Command $Arguments returned a non zero exit code ${LASTEXITCODE}." + } +} + +function Start-ExecuteWithRetry { + Param( + [Parameter(Mandatory=$true)] + [ScriptBlock]$ScriptBlock, + [Int]$MaxRetryCount=10, + [Int]$RetryInterval=3, + [String]$RetryMessage, + [Array]$ArgumentList=@() + ) + $currentErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $retryCount = 0 + while ($true) { + try { + $res = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList + $ErrorActionPreference = $currentErrorActionPreference + return $res + } catch [System.Exception] { + $retryCount++ + if ($retryCount -gt $MaxRetryCount) { + $ErrorActionPreference = $currentErrorActionPreference + Throw $_ + } else { + $prefixMsg = "Retry(${retryCount}/${MaxRetryCount})" + if($RetryMessage) { + Write-Host "${prefixMsg} - $RetryMessage" + } elseif($_) { + Write-Host "${prefixMsg} - $($_.ToString())" + } + Start-Sleep $RetryInterval + } + } + } +} + +function Start-FileDownload { + Param( + [Parameter(Mandatory=$true)] + [String]$URL, + [Parameter(Mandatory=$true)] + [String]$Destination, + [Int]$RetryCount=10 + ) + Write-Output "Downloading $URL to $Destination" + Start-ExecuteWithRetry ` + -ScriptBlock { Invoke-CommandLine -Command "curl.exe" -Arguments "-L -s -o $Destination $URL" } ` + -MaxRetryCount $RetryCount ` + -RetryMessage "Failed to download '${URL}'. Retrying" + Write-Output "Successfully downloaded." +} + +function Add-ToPathEnvVar { + Param( + [Parameter(Mandatory=$true)] + [String[]]$Path, + [Parameter(Mandatory=$false)] + [ValidateSet([System.EnvironmentVariableTarget]::User, [System.EnvironmentVariableTarget]::Machine)] + [System.EnvironmentVariableTarget]$Target=[System.EnvironmentVariableTarget]::Machine + ) + $pathEnvVar = [Environment]::GetEnvironmentVariable("PATH", $Target).Split(';') + $currentSessionPath = $env:PATH.Split(';') + foreach($p in $Path) { + if($p -notin $pathEnvVar) { + $pathEnvVar += $p + } + if($p -notin $currentSessionPath) { + $currentSessionPath += $p + } + } + $env:PATH = $currentSessionPath -join ';' + $newPathEnvVar = $pathEnvVar -join ';' + [Environment]::SetEnvironmentVariable("PATH", $newPathEnvVar, $Target) +} + +function Install-Tool { + [CmdletBinding(DefaultParameterSetName = "URL")] + Param( + [Parameter(Mandatory=$true, ParameterSetName = "URL")] + [String]$URL, + [Parameter(Mandatory=$true, ParameterSetName = "LocalPath")] + [String]$LocalPath, + [Parameter(ParameterSetName = "URL")] + [Parameter(ParameterSetName = "LocalPath")] + [String[]]$Params=@(), + [Parameter(ParameterSetName = "URL")] + [Parameter(ParameterSetName = "LocalPath")] + [Int[]]$AllowedExitCodes=@(0) + ) + PROCESS { + $installerPath = $LocalPath + if($PSCmdlet.ParameterSetName -eq "URL") { + $installerPath = Join-Path $env:TEMP $URL.Split('/')[-1] + Start-FileDownload -URL $URL -Destination $installerPath + } + Write-Output "Installing ${installerPath}" + $kwargs = @{ + "FilePath" = $installerPath + "ArgumentList" = $Params + "NoNewWindow" = $true + "PassThru" = $true + "Wait" = $true + } + if((Get-ChildItem $installerPath).Extension -eq '.msi') { + $kwargs["FilePath"] = "msiexec.exe" + $kwargs["ArgumentList"] = @("/i", $installerPath) + $Params + } + $p = Start-Process @kwargs + if($p.ExitCode -notin $AllowedExitCodes) { + Throw "Installation failed. Exit code: $($p.ExitCode)" + } + if($PSCmdlet.ParameterSetName -eq "URL") { + Start-ExecuteWithRetry ` + -ScriptBlock { Remove-Item -Force -Path $installerPath -ErrorAction Stop } ` + -RetryMessage "Failed to remove ${installerPath}. Retrying" + } + } +} |