diff options
Diffstat (limited to 'src/bootstrap/metrics.rs')
| -rw-r--r-- | src/bootstrap/metrics.rs | 208 |
1 files changed, 208 insertions, 0 deletions
diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs new file mode 100644 index 000000000..451febddc --- /dev/null +++ b/src/bootstrap/metrics.rs @@ -0,0 +1,208 @@ +//! This module is responsible for collecting metrics profiling information for the current build +//! and dumping it to disk as JSON, to aid investigations on build and CI performance. +//! +//! As this module requires additional dependencies not present during local builds, it's cfg'd +//! away whenever the `build.metrics` config option is not set to `true`. + +use crate::builder::Step; +use crate::util::t; +use crate::Build; +use serde::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::fs::File; +use std::io::BufWriter; +use std::time::{Duration, Instant}; +use sysinfo::{CpuExt, System, SystemExt}; + +pub(crate) struct BuildMetrics { + state: RefCell<MetricsState>, +} + +impl BuildMetrics { + pub(crate) fn init() -> Self { + let state = RefCell::new(MetricsState { + finished_steps: Vec::new(), + running_steps: Vec::new(), + + system_info: System::new(), + timer_start: None, + invocation_timer_start: Instant::now(), + }); + + BuildMetrics { state } + } + + pub(crate) fn enter_step<S: Step>(&self, step: &S) { + let mut state = self.state.borrow_mut(); + + // Consider all the stats gathered so far as the parent's. + if !state.running_steps.is_empty() { + self.collect_stats(&mut *state); + } + + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + + state.running_steps.push(StepMetrics { + type_: std::any::type_name::<S>().into(), + debug_repr: format!("{step:?}"), + + cpu_usage_time_sec: 0.0, + duration_excluding_children_sec: Duration::ZERO, + + children: Vec::new(), + }); + } + + pub(crate) fn exit_step(&self) { + let mut state = self.state.borrow_mut(); + + self.collect_stats(&mut *state); + + let step = state.running_steps.pop().unwrap(); + if state.running_steps.is_empty() { + state.finished_steps.push(step); + state.timer_start = None; + } else { + state.running_steps.last_mut().unwrap().children.push(step); + + // Start collecting again for the parent step. + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + } + } + + fn collect_stats(&self, state: &mut MetricsState) { + let step = state.running_steps.last_mut().unwrap(); + + let elapsed = state.timer_start.unwrap().elapsed(); + step.duration_excluding_children_sec += elapsed; + + state.system_info.refresh_cpu(); + let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::<f32>(); + step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); + } + + pub(crate) fn persist(&self, build: &Build) { + let mut state = self.state.borrow_mut(); + assert!(state.running_steps.is_empty(), "steps are still executing"); + + let dest = build.out.join("metrics.json"); + + let mut system = System::new(); + system.refresh_cpu(); + system.refresh_memory(); + + let system_stats = JsonInvocationSystemStats { + cpu_threads_count: system.cpus().len(), + cpu_model: system.cpus()[0].brand().into(), + + memory_total_bytes: system.total_memory() * 1024, + }; + let steps = std::mem::take(&mut state.finished_steps); + + // Some of our CI builds consist of multiple independent CI invocations. Ensure all the + // previous invocations are still present in the resulting file. + let mut invocations = match std::fs::read(&dest) { + Ok(contents) => t!(serde_json::from_slice::<JsonRoot>(&contents)).invocations, + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + panic!("failed to open existing metrics file at {}: {err}", dest.display()); + } + Vec::new() + } + }; + invocations.push(JsonInvocation { + duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), + children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), + }); + + let json = JsonRoot { system_stats, invocations }; + + t!(std::fs::create_dir_all(dest.parent().unwrap())); + let mut file = BufWriter::new(t!(File::create(&dest))); + t!(serde_json::to_writer(&mut file, &json)); + } + + fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { + JsonNode::RustbuildStep { + type_: step.type_, + debug_repr: step.debug_repr, + + duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), + system_stats: JsonStepSystemStats { + cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 + / step.duration_excluding_children_sec.as_secs_f64(), + }, + + children: step + .children + .into_iter() + .map(|child| self.prepare_json_step(child)) + .collect(), + } + } +} + +struct MetricsState { + finished_steps: Vec<StepMetrics>, + running_steps: Vec<StepMetrics>, + + system_info: System, + timer_start: Option<Instant>, + invocation_timer_start: Instant, +} + +struct StepMetrics { + type_: String, + debug_repr: String, + + cpu_usage_time_sec: f64, + duration_excluding_children_sec: Duration, + + children: Vec<StepMetrics>, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonRoot { + system_stats: JsonInvocationSystemStats, + invocations: Vec<JsonInvocation>, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonInvocation { + duration_including_children_sec: f64, + children: Vec<JsonNode>, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum JsonNode { + RustbuildStep { + #[serde(rename = "type")] + type_: String, + debug_repr: String, + + duration_excluding_children_sec: f64, + system_stats: JsonStepSystemStats, + + children: Vec<JsonNode>, + }, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonInvocationSystemStats { + cpu_threads_count: usize, + cpu_model: String, + + memory_total_bytes: u64, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +struct JsonStepSystemStats { + cpu_utilization_percent: f64, +} |