use crate::process_error::ProcessError; use crate::read2; use anyhow::{bail, Context, Result}; use jobserver::Client; use shell_escape::escape; use tempfile::NamedTempFile; use std::collections::BTreeMap; use std::env; use std::ffi::{OsStr, OsString}; use std::fmt; use std::io::{self, Write}; use std::iter::once; use std::path::Path; use std::process::{Command, ExitStatus, Output, Stdio}; /// A builder object for an external process, similar to [`std::process::Command`]. #[derive(Clone, Debug)] pub struct ProcessBuilder { /// The program to execute. program: OsString, /// A list of arguments to pass to the program. args: Vec, /// Any environment variables that should be set for the program. env: BTreeMap>, /// The directory to run the program from. cwd: Option, /// A list of wrappers that wrap the original program when calling /// [`ProcessBuilder::wrapped`]. The last one is the outermost one. wrappers: Vec, /// The `make` jobserver. See the [jobserver crate] for /// more information. /// /// [jobserver crate]: https://docs.rs/jobserver/ jobserver: Option, /// `true` to include environment variable in display. display_env_vars: bool, /// `true` to retry with an argfile if hitting "command line too big" error. /// See [`ProcessBuilder::retry_with_argfile`] for more information. retry_with_argfile: bool, /// Data to write to stdin. stdin: Option>, } impl fmt::Display for ProcessBuilder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "`")?; if self.display_env_vars { for (key, val) in self.env.iter() { if let Some(val) = val { let val = escape(val.to_string_lossy()); if cfg!(windows) { write!(f, "set {}={}&& ", key, val)?; } else { write!(f, "{}={} ", key, val)?; } } } } write!(f, "{}", self.get_program().to_string_lossy())?; for arg in self.get_args() { write!(f, " {}", escape(arg.to_string_lossy()))?; } write!(f, "`") } } impl ProcessBuilder { /// Creates a new [`ProcessBuilder`] with the given executable path. pub fn new>(cmd: T) -> ProcessBuilder { ProcessBuilder { program: cmd.as_ref().to_os_string(), args: Vec::new(), cwd: None, env: BTreeMap::new(), wrappers: Vec::new(), jobserver: None, display_env_vars: false, retry_with_argfile: false, stdin: None, } } /// (chainable) Sets the executable for the process. pub fn program>(&mut self, program: T) -> &mut ProcessBuilder { self.program = program.as_ref().to_os_string(); self } /// (chainable) Adds `arg` to the args list. pub fn arg>(&mut self, arg: T) -> &mut ProcessBuilder { self.args.push(arg.as_ref().to_os_string()); self } /// (chainable) Adds multiple `args` to the args list. pub fn args>(&mut self, args: &[T]) -> &mut ProcessBuilder { self.args .extend(args.iter().map(|t| t.as_ref().to_os_string())); self } /// (chainable) Replaces the args list with the given `args`. pub fn args_replace>(&mut self, args: &[T]) -> &mut ProcessBuilder { if let Some(program) = self.wrappers.pop() { // User intend to replace all args, so we // - use the outermost wrapper as the main program, and // - cleanup other inner wrappers. self.program = program; self.wrappers = Vec::new(); } self.args = args.iter().map(|t| t.as_ref().to_os_string()).collect(); self } /// (chainable) Sets the current working directory of the process. pub fn cwd>(&mut self, path: T) -> &mut ProcessBuilder { self.cwd = Some(path.as_ref().to_os_string()); self } /// (chainable) Sets an environment variable for the process. pub fn env>(&mut self, key: &str, val: T) -> &mut ProcessBuilder { self.env .insert(key.to_string(), Some(val.as_ref().to_os_string())); self } /// (chainable) Unsets an environment variable for the process. pub fn env_remove(&mut self, key: &str) -> &mut ProcessBuilder { self.env.insert(key.to_string(), None); self } /// Gets the executable name. pub fn get_program(&self) -> &OsString { self.wrappers.last().unwrap_or(&self.program) } /// Gets the program arguments. pub fn get_args(&self) -> impl Iterator { self.wrappers .iter() .rev() .chain(once(&self.program)) .chain(self.args.iter()) .skip(1) // Skip the main `program } /// Gets the current working directory for the process. pub fn get_cwd(&self) -> Option<&Path> { self.cwd.as_ref().map(Path::new) } /// Gets an environment variable as the process will see it (will inherit from environment /// unless explicitally unset). pub fn get_env(&self, var: &str) -> Option { self.env .get(var) .cloned() .or_else(|| Some(env::var_os(var))) .and_then(|s| s) } /// Gets all environment variables explicitly set or unset for the process (not inherited /// vars). pub fn get_envs(&self) -> &BTreeMap> { &self.env } /// Sets the `make` jobserver. See the [jobserver crate][jobserver_docs] for /// more information. /// /// [jobserver_docs]: https://docs.rs/jobserver/0.1.6/jobserver/ pub fn inherit_jobserver(&mut self, jobserver: &Client) -> &mut Self { self.jobserver = Some(jobserver.clone()); self } /// Enables environment variable display. pub fn display_env_vars(&mut self) -> &mut Self { self.display_env_vars = true; self } /// Enables retrying with an argfile if hitting "command line too big" error /// /// This is primarily for the `@path` arg of rustc and rustdoc, which treat /// each line as an command-line argument, so `LF` and `CRLF` bytes are not /// valid as an argument for argfile at this moment. /// For example, `RUSTDOCFLAGS="--crate-version foo\nbar" cargo doc` is /// valid when invoking from command-line but not from argfile. /// /// To sum up, the limitations of the argfile are: /// /// - Must be valid UTF-8 encoded. /// - Must not contain any newlines in each argument. /// /// Ref: /// /// - /// - pub fn retry_with_argfile(&mut self, enabled: bool) -> &mut Self { self.retry_with_argfile = enabled; self } /// Sets a value that will be written to stdin of the process on launch. pub fn stdin>>(&mut self, stdin: T) -> &mut Self { self.stdin = Some(stdin.into()); self } fn should_retry_with_argfile(&self, err: &io::Error) -> bool { self.retry_with_argfile && imp::command_line_too_big(err) } /// Like [`Command::status`] but with a better error message. pub fn status(&self) -> Result { self._status() .with_context(|| ProcessError::could_not_execute(self)) } fn _status(&self) -> io::Result { if !debug_force_argfile(self.retry_with_argfile) { let mut cmd = self.build_command(); match cmd.spawn() { Err(ref e) if self.should_retry_with_argfile(e) => {} Err(e) => return Err(e), Ok(mut child) => return child.wait(), } } let (mut cmd, argfile) = self.build_command_with_argfile()?; let status = cmd.spawn()?.wait(); close_tempfile_and_log_error(argfile); status } /// Runs the process, waiting for completion, and mapping non-success exit codes to an error. pub fn exec(&self) -> Result<()> { let exit = self.status()?; if exit.success() { Ok(()) } else { Err(ProcessError::new( &format!("process didn't exit successfully: {}", self), Some(exit), None, ) .into()) } } /// Replaces the current process with the target process. /// /// On Unix, this executes the process using the Unix syscall `execvp`, which will block /// this process, and will only return if there is an error. /// /// On Windows this isn't technically possible. Instead we emulate it to the best of our /// ability. One aspect we fix here is that we specify a handler for the Ctrl-C handler. /// In doing so (and by effectively ignoring it) we should emulate proxying Ctrl-C /// handling to the application at hand, which will either terminate or handle it itself. /// According to Microsoft's documentation at /// . /// the Ctrl-C signal is sent to all processes attached to a terminal, which should /// include our child process. If the child terminates then we'll reap them in Cargo /// pretty quickly, and if the child handles the signal then we won't terminate /// (and we shouldn't!) until the process itself later exits. pub fn exec_replace(&self) -> Result<()> { imp::exec_replace(self) } /// Like [`Command::output`] but with a better error message. pub fn output(&self) -> Result { self._output() .with_context(|| ProcessError::could_not_execute(self)) } fn _output(&self) -> io::Result { if !debug_force_argfile(self.retry_with_argfile) { let mut cmd = self.build_command(); match piped(&mut cmd, self.stdin.is_some()).spawn() { Err(ref e) if self.should_retry_with_argfile(e) => {} Err(e) => return Err(e), Ok(mut child) => { if let Some(stdin) = &self.stdin { child.stdin.take().unwrap().write_all(stdin)?; } return child.wait_with_output(); } } } let (mut cmd, argfile) = self.build_command_with_argfile()?; let mut child = piped(&mut cmd, self.stdin.is_some()).spawn()?; if let Some(stdin) = &self.stdin { child.stdin.take().unwrap().write_all(stdin)?; } let output = child.wait_with_output(); close_tempfile_and_log_error(argfile); output } /// Executes the process, returning the stdio output, or an error if non-zero exit status. pub fn exec_with_output(&self) -> Result { let output = self.output()?; if output.status.success() { Ok(output) } else { Err(ProcessError::new( &format!("process didn't exit successfully: {}", self), Some(output.status), Some(&output), ) .into()) } } /// Executes a command, passing each line of stdout and stderr to the supplied callbacks, which /// can mutate the string data. /// /// If any invocations of these function return an error, it will be propagated. /// /// If `capture_output` is true, then all the output will also be buffered /// and stored in the returned `Output` object. If it is false, no caching /// is done, and the callbacks are solely responsible for handling the /// output. pub fn exec_with_streaming( &self, on_stdout_line: &mut dyn FnMut(&str) -> Result<()>, on_stderr_line: &mut dyn FnMut(&str) -> Result<()>, capture_output: bool, ) -> Result { let mut stdout = Vec::new(); let mut stderr = Vec::new(); let mut callback_error = None; let mut stdout_pos = 0; let mut stderr_pos = 0; let spawn = |mut cmd| { if !debug_force_argfile(self.retry_with_argfile) { match piped(&mut cmd, false).spawn() { Err(ref e) if self.should_retry_with_argfile(e) => {} Err(e) => return Err(e), Ok(child) => return Ok((child, None)), } } let (mut cmd, argfile) = self.build_command_with_argfile()?; Ok((piped(&mut cmd, false).spawn()?, Some(argfile))) }; let status = (|| { let cmd = self.build_command(); let (mut child, argfile) = spawn(cmd)?; let out = child.stdout.take().unwrap(); let err = child.stderr.take().unwrap(); read2(out, err, &mut |is_out, data, eof| { let pos = if is_out { &mut stdout_pos } else { &mut stderr_pos }; let idx = if eof { data.len() } else { match data[*pos..].iter().rposition(|b| *b == b'\n') { Some(i) => *pos + i + 1, None => { *pos = data.len(); return; } } }; let new_lines = &data[..idx]; for line in String::from_utf8_lossy(new_lines).lines() { if callback_error.is_some() { break; } let callback_result = if is_out { on_stdout_line(line) } else { on_stderr_line(line) }; if let Err(e) = callback_result { callback_error = Some(e); break; } } if capture_output { let dst = if is_out { &mut stdout } else { &mut stderr }; dst.extend(new_lines); } data.drain(..idx); *pos = 0; })?; let status = child.wait(); if let Some(argfile) = argfile { close_tempfile_and_log_error(argfile); } status })() .with_context(|| ProcessError::could_not_execute(self))?; let output = Output { status, stdout, stderr, }; { let to_print = if capture_output { Some(&output) } else { None }; if let Some(e) = callback_error { let cx = ProcessError::new( &format!("failed to parse process output: {}", self), Some(output.status), to_print, ); bail!(anyhow::Error::new(cx).context(e)); } else if !output.status.success() { bail!(ProcessError::new( &format!("process didn't exit successfully: {}", self), Some(output.status), to_print, )); } } Ok(output) } /// Builds the command with an `@` argfile that contains all the /// arguments. This is primarily served for rustc/rustdoc command family. fn build_command_with_argfile(&self) -> io::Result<(Command, NamedTempFile)> { use std::io::Write as _; let mut tmp = tempfile::Builder::new() .prefix("cargo-argfile.") .tempfile()?; let mut arg = OsString::from("@"); arg.push(tmp.path()); let mut cmd = self.build_command_without_args(); cmd.arg(arg); log::debug!("created argfile at {} for {self}", tmp.path().display()); let cap = self.get_args().map(|arg| arg.len() + 1).sum::(); let mut buf = Vec::with_capacity(cap); for arg in &self.args { let arg = arg.to_str().ok_or_else(|| { io::Error::new( io::ErrorKind::Other, format!( "argument for argfile contains invalid UTF-8 characters: `{}`", arg.to_string_lossy() ), ) })?; if arg.contains('\n') { return Err(io::Error::new( io::ErrorKind::Other, format!("argument for argfile contains newlines: `{arg}`"), )); } writeln!(buf, "{arg}")?; } tmp.write_all(&mut buf)?; Ok((cmd, tmp)) } /// Builds a command from `ProcessBuilder` for everything but not `args`. fn build_command_without_args(&self) -> Command { let mut command = { let mut iter = self.wrappers.iter().rev().chain(once(&self.program)); let mut cmd = Command::new(iter.next().expect("at least one `program` exists")); cmd.args(iter); cmd }; if let Some(cwd) = self.get_cwd() { command.current_dir(cwd); } for (k, v) in &self.env { match *v { Some(ref v) => { command.env(k, v); } None => { command.env_remove(k); } } } if let Some(ref c) = self.jobserver { c.configure(&mut command); } command } /// Converts `ProcessBuilder` into a `std::process::Command`, and handles /// the jobserver, if present. /// /// Note that this method doesn't take argfile fallback into account. The /// caller should handle it by themselves. pub fn build_command(&self) -> Command { let mut command = self.build_command_without_args(); for arg in &self.args { command.arg(arg); } command } /// Wraps an existing command with the provided wrapper, if it is present and valid. /// /// # Examples /// /// ```rust /// use cargo_util::ProcessBuilder; /// // Running this would execute `rustc` /// let cmd = ProcessBuilder::new("rustc"); /// /// // Running this will execute `sccache rustc` /// let cmd = cmd.wrapped(Some("sccache")); /// ``` pub fn wrapped(mut self, wrapper: Option>) -> Self { if let Some(wrapper) = wrapper.as_ref() { let wrapper = wrapper.as_ref(); if !wrapper.is_empty() { self.wrappers.push(wrapper.to_os_string()); } } self } } /// Forces the command to use `@path` argfile. /// /// You should set `__CARGO_TEST_FORCE_ARGFILE` to enable this. fn debug_force_argfile(retry_enabled: bool) -> bool { cfg!(debug_assertions) && env::var("__CARGO_TEST_FORCE_ARGFILE").is_ok() && retry_enabled } /// Creates new pipes for stderr, stdout, and optionally stdin. fn piped(cmd: &mut Command, pipe_stdin: bool) -> &mut Command { cmd.stdout(Stdio::piped()) .stderr(Stdio::piped()) .stdin(if pipe_stdin { Stdio::piped() } else { Stdio::null() }) } fn close_tempfile_and_log_error(file: NamedTempFile) { file.close().unwrap_or_else(|e| { log::warn!("failed to close temporary file: {e}"); }); } #[cfg(unix)] mod imp { use super::{close_tempfile_and_log_error, debug_force_argfile, ProcessBuilder, ProcessError}; use anyhow::Result; use std::io; use std::os::unix::process::CommandExt; pub fn exec_replace(process_builder: &ProcessBuilder) -> Result<()> { let mut error; let mut file = None; if debug_force_argfile(process_builder.retry_with_argfile) { let (mut command, argfile) = process_builder.build_command_with_argfile()?; file = Some(argfile); error = command.exec() } else { let mut command = process_builder.build_command(); error = command.exec(); if process_builder.should_retry_with_argfile(&error) { let (mut command, argfile) = process_builder.build_command_with_argfile()?; file = Some(argfile); error = command.exec() } } if let Some(file) = file { close_tempfile_and_log_error(file); } Err(anyhow::Error::from(error).context(ProcessError::new( &format!("could not execute process {}", process_builder), None, None, ))) } pub fn command_line_too_big(err: &io::Error) -> bool { err.raw_os_error() == Some(libc::E2BIG) } } #[cfg(windows)] mod imp { use super::{ProcessBuilder, ProcessError}; use anyhow::Result; use std::io; use windows_sys::Win32::Foundation::{BOOL, FALSE, TRUE}; use windows_sys::Win32::System::Console::SetConsoleCtrlHandler; unsafe extern "system" fn ctrlc_handler(_: u32) -> BOOL { // Do nothing; let the child process handle it. TRUE } pub fn exec_replace(process_builder: &ProcessBuilder) -> Result<()> { unsafe { if SetConsoleCtrlHandler(Some(ctrlc_handler), TRUE) == FALSE { return Err(ProcessError::new("Could not set Ctrl-C handler.", None, None).into()); } } // Just execute the process as normal. process_builder.exec() } pub fn command_line_too_big(err: &io::Error) -> bool { use windows_sys::Win32::Foundation::ERROR_FILENAME_EXCED_RANGE; err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE as i32) } } #[cfg(test)] mod tests { use super::ProcessBuilder; use std::fs; #[test] fn argfile_build_succeeds() { let mut cmd = ProcessBuilder::new("echo"); cmd.args(["foo", "bar"].as_slice()); let (cmd, argfile) = cmd.build_command_with_argfile().unwrap(); assert_eq!(cmd.get_program(), "echo"); let cmd_args: Vec<_> = cmd.get_args().map(|s| s.to_str().unwrap()).collect(); assert_eq!(cmd_args.len(), 1); assert!(cmd_args[0].starts_with("@")); assert!(cmd_args[0].contains("cargo-argfile.")); let buf = fs::read_to_string(argfile.path()).unwrap(); assert_eq!(buf, "foo\nbar\n"); } #[test] fn argfile_build_fails_if_arg_contains_newline() { let mut cmd = ProcessBuilder::new("echo"); cmd.arg("foo\n"); let err = cmd.build_command_with_argfile().unwrap_err(); assert_eq!( err.to_string(), "argument for argfile contains newlines: `foo\n`" ); } #[test] fn argfile_build_fails_if_arg_contains_invalid_utf8() { let mut cmd = ProcessBuilder::new("echo"); #[cfg(windows)] let invalid_arg = { use std::os::windows::prelude::*; std::ffi::OsString::from_wide(&[0x0066, 0x006f, 0xD800, 0x006f]) }; #[cfg(unix)] let invalid_arg = { use std::os::unix::ffi::OsStrExt; std::ffi::OsStr::from_bytes(&[0x66, 0x6f, 0x80, 0x6f]).to_os_string() }; cmd.arg(invalid_arg); let err = cmd.build_command_with_argfile().unwrap_err(); assert_eq!( err.to_string(), "argument for argfile contains invalid UTF-8 characters: `fo�o`" ); } }