Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 163 insertions, 0 deletions

diff --git a/src/ci/cpu-usage-over-time.py b/src/ci/cpu-usage-over-time.py
new file mode 100644
index 000000000..adfd895ea
--- /dev/null
+++ b/src/ci/cpu-usage-over-time.py
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# ignore-tidy-linelength
+
+# This is a small script that we use on CI to collect CPU usage statistics of
+# our builders. By seeing graphs of CPU usage over time we hope to correlate
+# that with possible improvements to Rust's own build system, ideally diagnosing
+# that either builders are always fully using their CPU resources or they're
+# idle for long stretches of time.
+#
+# This script is relatively simple, but it's platform specific. Each platform
+# (OSX/Windows/Linux) has a different way of calculating the current state of
+# CPU at a point in time. We then compare two captured states to determine the
+# percentage of time spent in one state versus another. The state capturing is
+# all platform-specific but the loop at the bottom is the cross platform part
+# that executes everywhere.
+#
+# # Viewing statistics
+#
+# All builders will upload their CPU statistics as CSV files to our S3 buckets.
+# These URLS look like:
+#
+#   https://$bucket.s3.amazonaws.com/rustc-builds/$commit/cpu-$builder.csv
+#
+# for example
+#
+#   https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/68baada19cd5340f05f0db15a3e16d6671609bcc/cpu-x86_64-apple.csv
+#
+# Each CSV file has two columns. The first is the timestamp of the measurement
+# and the second column is the % of idle cpu time in that time slice. Ideally
+# the second column is always zero.
+#
+# Once you've downloaded a file there's various ways to plot it and visualize
+# it. For command line usage you use the `src/etc/cpu-usage-over-time-plot.sh`
+# script in this repository.
+
+import datetime
+import sys
+import time
+
+# Python 3.3 changed the value of `sys.platform` on Linux from "linux2" to just
+# "linux". We check here with `.startswith` to keep compatibility with older
+# Python versions (especially Python 2.7).
+if sys.platform.startswith('linux'):
+    class State:
+        def __init__(self):
+            with open('/proc/stat', 'r') as file:
+                data = file.readline().split()
+            if data[0] != 'cpu':
+                raise Exception('did not start with "cpu"')
+            self.user = int(data[1])
+            self.nice = int(data[2])
+            self.system = int(data[3])
+            self.idle = int(data[4])
+            self.iowait = int(data[5])
+            self.irq = int(data[6])
+            self.softirq = int(data[7])
+            self.steal = int(data[8])
+            self.guest = int(data[9])
+            self.guest_nice = int(data[10])
+
+        def idle_since(self, prev):
+            user = self.user - prev.user
+            nice = self.nice - prev.nice
+            system = self.system - prev.system
+            idle = self.idle - prev.idle
+            iowait = self.iowait - prev.iowait
+            irq = self.irq - prev.irq
+            softirq = self.softirq - prev.softirq
+            steal = self.steal - prev.steal
+            guest = self.guest - prev.guest
+            guest_nice = self.guest_nice - prev.guest_nice
+            total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice
+            return float(idle) / float(total) * 100
+
+elif sys.platform == 'win32':
+    from ctypes.wintypes import DWORD
+    from ctypes import Structure, windll, WinError, GetLastError, byref
+
+    class FILETIME(Structure):
+        _fields_ = [
+            ("dwLowDateTime", DWORD),
+            ("dwHighDateTime", DWORD),
+        ]
+
+    class State:
+        def __init__(self):
+            idle, kernel, user = FILETIME(), FILETIME(), FILETIME()
+
+            success = windll.kernel32.GetSystemTimes(
+                byref(idle),
+                byref(kernel),
+                byref(user),
+            )
+
+            assert success, WinError(GetLastError())[1]
+
+            self.idle = (idle.dwHighDateTime << 32) | idle.dwLowDateTime
+            self.kernel = (kernel.dwHighDateTime << 32) | kernel.dwLowDateTime
+            self.user = (user.dwHighDateTime << 32) | user.dwLowDateTime
+
+        def idle_since(self, prev):
+            idle = self.idle - prev.idle
+            user = self.user - prev.user
+            kernel = self.kernel - prev.kernel
+            return float(idle) / float(user + kernel) * 100
+
+elif sys.platform == 'darwin':
+    from ctypes import *
+    libc = cdll.LoadLibrary('/usr/lib/libc.dylib')
+
+    class host_cpu_load_info_data_t(Structure):
+        _fields_ = [("cpu_ticks", c_uint * 4)]
+
+    host_statistics = libc.host_statistics
+    host_statistics.argtypes = [
+        c_uint,
+        c_int,
+        POINTER(host_cpu_load_info_data_t),
+        POINTER(c_int)
+    ]
+    host_statistics.restype = c_int
+
+    CPU_STATE_USER = 0
+    CPU_STATE_SYSTEM = 1
+    CPU_STATE_IDLE = 2
+    CPU_STATE_NICE = 3
+    class State:
+        def __init__(self):
+            stats = host_cpu_load_info_data_t()
+            count = c_int(4) # HOST_CPU_LOAD_INFO_COUNT
+            err = libc.host_statistics(
+                libc.mach_host_self(),
+                c_int(3), # HOST_CPU_LOAD_INFO
+                byref(stats),
+                byref(count),
+            )
+            assert err == 0
+            self.system = stats.cpu_ticks[CPU_STATE_SYSTEM]
+            self.user = stats.cpu_ticks[CPU_STATE_USER]
+            self.idle = stats.cpu_ticks[CPU_STATE_IDLE]
+            self.nice = stats.cpu_ticks[CPU_STATE_NICE]
+
+        def idle_since(self, prev):
+            user = self.user - prev.user
+            system = self.system - prev.system
+            idle = self.idle - prev.idle
+            nice = self.nice - prev.nice
+            return float(idle) / float(user + system + idle + nice) * 100.0
+
+else:
+    print('unknown platform', sys.platform)
+    sys.exit(1)
+
+cur_state = State()
+print("Time,Idle")
+while True:
+    time.sleep(1)
+    next_state = State()
+    now = datetime.datetime.utcnow().isoformat()
+    idle = next_state.idle_since(cur_state)
+    print("%s,%s" % (now, idle))
+    sys.stdout.flush()
+    cur_state = next_state