Adding upstream version 1:3.11.1.upstream/1%3.11.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 1417 insertions, 0 deletions

diff --git a/lmr/lmr.h b/lmr/lmr.h
new file mode 100644
index 0000000..7375c33
--- /dev/null
+++ b/lmr/lmr.h
@@ -0,0 +1,228 @@
+/*
+ *	The PCI Utilities -- Margining utility main header
+ *
+ *	Copyright (c) 2023 KNS Group LLC (YADRO)
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL v2+.
+ *
+ *	SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef _LMR_H
+#define _LMR_H
+
+#include <stdbool.h>
+
+#include "pciutils.h"
+
+#define MARGIN_STEP_MS 1000
+
+#define MARGIN_TIM_MIN       20
+#define MARGIN_TIM_RECOMMEND 30
+#define MARGIN_VOLT_MIN      50
+
+enum margin_hw { MARGIN_HW_DEFAULT, MARGIN_ICE_LAKE_RC };
+
+/* PCI Device wrapper for margining functions */
+struct margin_dev {
+  struct pci_dev *dev;
+  int lmr_cap_addr;
+  u8 width;
+  u8 retimers_n;
+  u8 link_speed;
+
+  enum margin_hw hw;
+
+  /* Saved Device settings to restore after margining */
+  u8 aspm;
+  bool hasd; // Hardware Autonomous Speed Disable
+  bool hawd; // Hardware Autonomous Width Disable
+};
+
+struct margin_link {
+  struct margin_dev down_port;
+  struct margin_dev up_port;
+};
+
+/* Specification Revision 5.0 Table 8-11 */
+struct margin_params {
+  bool ind_error_sampler;
+  bool sample_report_method;
+  bool ind_left_right_tim;
+  bool ind_up_down_volt;
+  bool volt_support;
+
+  u8 max_lanes;
+
+  u8 timing_steps;
+  u8 timing_offset;
+
+  u8 volt_steps;
+  u8 volt_offset;
+
+  u8 sample_rate_v;
+  u8 sample_rate_t;
+};
+
+/* Step Margin Execution Status - Step command response */
+enum margin_step_exec_sts {
+  MARGIN_NAK = 0, // NAK/Set up for margin
+  MARGIN_LIM,     // Too many errors (device limit)
+  MARGIN_THR      // Test threshold has been reached
+};
+
+enum margin_dir { VOLT_UP = 0, VOLT_DOWN, TIM_LEFT, TIM_RIGHT };
+
+/* Margining results of one lane of the receiver */
+struct margin_res_lane {
+  u8 lane;
+  u8 steps[4];
+  enum margin_step_exec_sts statuses[4];
+};
+
+/* Reason not to run margining test on the Link/Receiver */
+enum margin_test_status {
+  MARGIN_TEST_OK = 0,
+  MARGIN_TEST_READY_BIT,
+  MARGIN_TEST_CAPS,
+
+  // Couldn't run test
+  MARGIN_TEST_PREREQS,
+  MARGIN_TEST_ARGS_LANES,
+  MARGIN_TEST_ARGS_RECVS,
+  MARGIN_TEST_ASPM
+};
+
+/* All lanes Receiver results */
+struct margin_results {
+  u8 recvn; // Receiver Number
+  struct margin_params params;
+  bool lane_reversal;
+  u8 link_speed;
+
+  enum margin_test_status test_status;
+
+  /* Used to convert steps to physical quantity.
+     Calculated from MaxOffset and NumSteps     */
+  double tim_coef;
+  double volt_coef;
+
+  bool tim_off_reported;
+  bool volt_off_reported;
+
+  u8 lanes_n;
+  struct margin_res_lane *lanes;
+};
+
+/* pcilmr arguments */
+struct margin_args {
+  u8 steps_t;        // 0 == use NumTimingSteps
+  u8 steps_v;        // 0 == use NumVoltageSteps
+  u8 parallel_lanes; // [1; MaxLanes + 1]
+  u8 error_limit;    // [0; 63]
+  u8 recvs[6];       // Receivers Numbers
+  u8 recvs_n;        // 0 == margin all available receivers
+  u8 lanes[32];      // Lanes to Margin
+  u8 lanes_n;        // 0 == margin all available lanes
+  bool run_margin;   // Or print params only
+  u8 verbosity;      // 0 - basic;
+                     // 1 - add info about remaining time and lanes in progress during margining
+
+  u64 *steps_utility; // For ETA logging
+};
+
+/* Receiver structure */
+struct margin_recv {
+  struct margin_dev *dev;
+  u8 recvn; // Receiver Number
+  bool lane_reversal;
+  struct margin_params *params;
+
+  u8 parallel_lanes;
+  u8 error_limit;
+};
+
+struct margin_lanes_data {
+  struct margin_recv *recv;
+
+  struct margin_res_lane *results;
+  u8 *lanes_numbers;
+  u8 lanes_n;
+
+  bool ind;
+  enum margin_dir dir;
+
+  u8 steps_lane_done;
+  u8 steps_lane_total;
+  u64 *steps_utility;
+
+  u8 verbosity;
+};
+
+/* margin_hw */
+
+/* Verify that devices form the link with 16 GT/s or 32 GT/s data rate */
+bool margin_verify_link(struct pci_dev *down_port, struct pci_dev *up_port);
+
+/* Check Margining Ready bit from Margining Port Status Register */
+bool margin_check_ready_bit(struct pci_dev *dev);
+
+/* Verify link and fill wrappers */
+bool margin_fill_link(struct pci_dev *down_port, struct pci_dev *up_port,
+                      struct margin_link *wrappers);
+
+/* Disable ASPM, set Hardware Autonomous Speed/Width Disable bits */
+bool margin_prep_link(struct margin_link *link);
+
+/* Restore ASPM, Hardware Autonomous Speed/Width settings */
+void margin_restore_link(struct margin_link *link);
+
+/* margin */
+
+/* Fill margin_params without calling other functions */
+bool margin_read_params(struct pci_access *pacc, struct pci_dev *dev, u8 recvn,
+                        struct margin_params *params);
+
+enum margin_test_status margin_process_args(struct margin_dev *dev, struct margin_args *args);
+
+/* Awaits that args are prepared through process_args.
+   Returns number of margined Receivers through recvs_n */
+struct margin_results *margin_test_link(struct margin_link *link, struct margin_args *args,
+                                        u8 *recvs_n);
+
+void margin_free_results(struct margin_results *results, u8 results_n);
+
+/* margin_log */
+
+extern bool margin_global_logging;
+extern bool margin_print_domain;
+
+void margin_log(char *format, ...);
+
+/* b:d.f -> b:d.f */
+void margin_log_bdfs(struct pci_dev *down_port, struct pci_dev *up_port);
+
+/* Print Link header (bdfs, width, speed) */
+void margin_log_link(struct margin_link *link);
+
+void margin_log_params(struct margin_params *params);
+
+/* Print receiver number */
+void margin_log_recvn(struct margin_recv *recv);
+
+/* Print full info from Receiver struct */
+void margin_log_receiver(struct margin_recv *recv);
+
+/* Margining in progress log */
+void margin_log_margining(struct margin_lanes_data arg);
+
+void margin_log_hw_quirks(struct margin_recv *recv);
+
+/* margin_results */
+
+void margin_results_print_brief(struct margin_results *results, u8 recvs_n);
+
+void margin_results_save_csv(struct margin_results *results, u8 recvs_n, char *dir,
+                             struct pci_dev *up_port);
+
+#endif
diff --git a/lmr/margin.c b/lmr/margin.c
new file mode 100644
index 0000000..a8c6571
--- /dev/null
+++ b/lmr/margin.c
@@ -0,0 +1,588 @@
+/*
+ *	The PCI Utilities -- Obtain the margin information of the Link
+ *
+ *	Copyright (c) 2023 KNS Group LLC (YADRO)
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL v2+.
+ *
+ *	SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include <time.h>
+
+#include "lmr.h"
+
+#ifdef PCI_OS_DJGPP
+#include <unistd.h>
+#endif
+
+/* Macro helpers for Margining command parsing */
+
+typedef u16 margin_cmd;
+
+/* Margining command parsing */
+
+#define LMR_CMD_RECVN   MASK(2, 0)
+#define LMR_CMD_TYPE    MASK(5, 3)
+#define LMR_CMD_PAYLOAD MASK(15, 8)
+
+// Payload parsing
+
+// Report Capabilities
+#define LMR_PLD_VOLT_SUPPORT         BIT(8)
+#define LMR_PLD_IND_U_D_VOLT         BIT(9)
+#define LMR_PLD_IND_L_R_TIM          BIT(10)
+#define LMR_PLD_SAMPLE_REPORT_METHOD BIT(11)
+#define LMR_PLD_IND_ERR_SAMPLER      BIT(12)
+
+#define LMR_PLD_MAX_T_STEPS MASK(13, 8)
+#define LMR_PLD_MAX_V_STEPS MASK(14, 8)
+#define LMR_PLD_MAX_OFFSET  MASK(14, 8)
+#define LMR_PLD_MAX_LANES   MASK(12, 8)
+#define LMR_PLD_SAMPLE_RATE MASK(13, 8)
+
+// Timing Step
+#define LMR_PLD_MARGIN_T_STEPS MASK(13, 8)
+#define LMR_PLD_T_GO_LEFT      BIT(14)
+
+// Voltage Timing
+#define LMR_PLD_MARGIN_V_STEPS MASK(14, 8)
+#define LMR_PLD_V_GO_DOWN      BIT(15)
+
+// Step Response
+#define LMR_PLD_ERR_CNT    MASK(13, 8)
+#define LMR_PLD_MARGIN_STS MASK(15, 14)
+
+/* Address calc macro for Lanes Margining registers */
+
+#define LMR_LANE_CTRL(lmr_cap_addr, lane)   ((lmr_cap_addr) + 8 + 4 * (lane))
+#define LMR_LANE_STATUS(lmr_cap_addr, lane) ((lmr_cap_addr) + 10 + 4 * (lane))
+
+/* Margining Commands */
+
+#define MARG_TIM(go_left, step, recvn)  margin_make_cmd(((go_left) << 6) | (step), 3, recvn)
+#define MARG_VOLT(go_down, step, recvn) margin_make_cmd(((go_down) << 7) | (step), 4, recvn)
+
+// Report commands
+#define REPORT_CAPS(recvn)         margin_make_cmd(0x88, 1, recvn)
+#define REPORT_VOL_STEPS(recvn)    margin_make_cmd(0x89, 1, recvn)
+#define REPORT_TIM_STEPS(recvn)    margin_make_cmd(0x8A, 1, recvn)
+#define REPORT_TIM_OFFSET(recvn)   margin_make_cmd(0x8B, 1, recvn)
+#define REPORT_VOL_OFFSET(recvn)   margin_make_cmd(0x8C, 1, recvn)
+#define REPORT_SAMPL_RATE_V(recvn) margin_make_cmd(0x8D, 1, recvn)
+#define REPORT_SAMPL_RATE_T(recvn) margin_make_cmd(0x8E, 1, recvn)
+#define REPORT_SAMPLE_CNT(recvn)   margin_make_cmd(0x8F, 1, recvn)
+#define REPORT_MAX_LANES(recvn)    margin_make_cmd(0x90, 1, recvn)
+
+// Set commands
+#define NO_COMMAND                          margin_make_cmd(0x9C, 7, 0)
+#define CLEAR_ERROR_LOG(recvn)              margin_make_cmd(0x55, 2, recvn)
+#define GO_TO_NORMAL_SETTINGS(recvn)        margin_make_cmd(0xF, 2, recvn)
+#define SET_ERROR_LIMIT(error_limit, recvn) margin_make_cmd(0xC0 | (error_limit), 2, recvn)
+
+static int
+msleep(long msec)
+{
+#if defined(PCI_OS_WINDOWS)
+  Sleep(msec);
+  return 0;
+#elif defined(PCI_OS_DJGPP)
+  if (msec * 1000 < 11264)
+    usleep(11264);
+  else
+    usleep(msec * 1000);
+  return 0;
+#else
+  struct timespec ts;
+  int res;
+
+  if (msec < 0)
+    {
+      errno = EINVAL;
+      return -1;
+    }
+
+  ts.tv_sec = msec / 1000;
+  ts.tv_nsec = (msec % 1000) * 1000000;
+
+  do
+    {
+      res = nanosleep(&ts, &ts);
+  } while (res && errno == EINTR);
+
+  return res;
+#endif
+}
+
+static margin_cmd
+margin_make_cmd(u8 payload, u8 type, u8 recvn)
+{
+  return SET_REG_MASK(0, LMR_CMD_PAYLOAD, payload) | SET_REG_MASK(0, LMR_CMD_TYPE, type)
+         | SET_REG_MASK(0, LMR_CMD_RECVN, recvn);
+}
+
+static bool
+margin_set_cmd(struct margin_dev *dev, u8 lane, margin_cmd cmd)
+{
+  pci_write_word(dev->dev, LMR_LANE_CTRL(dev->lmr_cap_addr, lane), cmd);
+  msleep(10);
+  return pci_read_word(dev->dev, LMR_LANE_STATUS(dev->lmr_cap_addr, lane)) == cmd;
+}
+
+static bool
+margin_report_cmd(struct margin_dev *dev, u8 lane, margin_cmd cmd, margin_cmd *result)
+{
+  pci_write_word(dev->dev, LMR_LANE_CTRL(dev->lmr_cap_addr, lane), cmd);
+  msleep(10);
+  *result = pci_read_word(dev->dev, LMR_LANE_STATUS(dev->lmr_cap_addr, lane));
+  return GET_REG_MASK(*result, LMR_CMD_TYPE) == GET_REG_MASK(cmd, LMR_CMD_TYPE)
+         && GET_REG_MASK(*result, LMR_CMD_RECVN) == GET_REG_MASK(cmd, LMR_CMD_RECVN)
+         && margin_set_cmd(dev, lane, NO_COMMAND);
+}
+
+static void
+margin_apply_hw_quirks(struct margin_recv *recv)
+{
+  switch (recv->dev->hw)
+    {
+      case MARGIN_ICE_LAKE_RC:
+        if (recv->recvn == 1)
+          recv->params->volt_offset = 12;
+        break;
+      default:
+        break;
+    }
+}
+
+static bool
+read_params_internal(struct margin_dev *dev, u8 recvn, bool lane_reversal,
+                     struct margin_params *params)
+{
+  margin_cmd resp;
+  u8 lane = lane_reversal ? dev->width - 1 : 0;
+  margin_set_cmd(dev, lane, NO_COMMAND);
+  bool status = margin_report_cmd(dev, lane, REPORT_CAPS(recvn), &resp);
+  if (status)
+    {
+      params->volt_support = GET_REG_MASK(resp, LMR_PLD_VOLT_SUPPORT);
+      params->ind_up_down_volt = GET_REG_MASK(resp, LMR_PLD_IND_U_D_VOLT);
+      params->ind_left_right_tim = GET_REG_MASK(resp, LMR_PLD_IND_L_R_TIM);
+      params->sample_report_method = GET_REG_MASK(resp, LMR_PLD_SAMPLE_REPORT_METHOD);
+      params->ind_error_sampler = GET_REG_MASK(resp, LMR_PLD_IND_ERR_SAMPLER);
+      status = margin_report_cmd(dev, lane, REPORT_VOL_STEPS(recvn), &resp);
+    }
+  if (status)
+    {
+      params->volt_steps = GET_REG_MASK(resp, LMR_PLD_MAX_V_STEPS);
+      status = margin_report_cmd(dev, lane, REPORT_TIM_STEPS(recvn), &resp);
+    }
+  if (status)
+    {
+      params->timing_steps = GET_REG_MASK(resp, LMR_PLD_MAX_T_STEPS);
+      status = margin_report_cmd(dev, lane, REPORT_TIM_OFFSET(recvn), &resp);
+    }
+  if (status)
+    {
+      params->timing_offset = GET_REG_MASK(resp, LMR_PLD_MAX_OFFSET);
+      status = margin_report_cmd(dev, lane, REPORT_VOL_OFFSET(recvn), &resp);
+    }
+  if (status)
+    {
+      params->volt_offset = GET_REG_MASK(resp, LMR_PLD_MAX_OFFSET);
+      status = margin_report_cmd(dev, lane, REPORT_SAMPL_RATE_V(recvn), &resp);
+    }
+  if (status)
+    {
+      params->sample_rate_v = GET_REG_MASK(resp, LMR_PLD_SAMPLE_RATE);
+      status = margin_report_cmd(dev, lane, REPORT_SAMPL_RATE_T(recvn), &resp);
+    }
+  if (status)
+    {
+      params->sample_rate_t = GET_REG_MASK(resp, LMR_PLD_SAMPLE_RATE);
+      status = margin_report_cmd(dev, lane, REPORT_MAX_LANES(recvn), &resp);
+    }
+  if (status)
+    params->max_lanes = GET_REG_MASK(resp, LMR_PLD_MAX_LANES);
+  return status;
+}
+
+/* Margin all lanes_n lanes simultaneously */
+static void
+margin_test_lanes(struct margin_lanes_data arg)
+{
+  u8 steps_done = 0;
+  margin_cmd lane_status;
+  u8 marg_type;
+  margin_cmd step_cmd;
+  bool timing = (arg.dir == TIM_LEFT || arg.dir == TIM_RIGHT);
+
+  if (timing)
+    {
+      marg_type = 3;
+      step_cmd = MARG_TIM(arg.dir == TIM_LEFT, steps_done, arg.recv->recvn);
+    }
+  else
+    {
+      marg_type = 4;
+      step_cmd = MARG_VOLT(arg.dir == VOLT_DOWN, steps_done, arg.recv->recvn);
+    }
+
+  bool failed_lanes[32] = { 0 };
+  u8 alive_lanes = arg.lanes_n;
+
+  for (int i = 0; i < arg.lanes_n; i++)
+    {
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane,
+                     SET_ERROR_LIMIT(arg.recv->error_limit, arg.recv->recvn));
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
+      arg.results[i].steps[arg.dir] = arg.steps_lane_total;
+      arg.results[i].statuses[arg.dir] = MARGIN_THR;
+    }
+
+  while (alive_lanes > 0 && steps_done < arg.steps_lane_total)
+    {
+      alive_lanes = 0;
+      steps_done++;
+      if (timing)
+        step_cmd = SET_REG_MASK(step_cmd, LMR_PLD_MARGIN_T_STEPS, steps_done);
+      else
+        step_cmd = SET_REG_MASK(step_cmd, LMR_PLD_MARGIN_V_STEPS, steps_done);
+
+      for (int i = 0; i < arg.lanes_n; i++)
+        {
+          if (!failed_lanes[i])
+            {
+              alive_lanes++;
+              int ctrl_addr = LMR_LANE_CTRL(arg.recv->dev->lmr_cap_addr, arg.results[i].lane);
+              pci_write_word(arg.recv->dev->dev, ctrl_addr, step_cmd);
+            }
+        }
+      msleep(MARGIN_STEP_MS);
+
+      for (int i = 0; i < arg.lanes_n; i++)
+        {
+          if (!failed_lanes[i])
+            {
+              int status_addr = LMR_LANE_STATUS(arg.recv->dev->lmr_cap_addr, arg.results[i].lane);
+              lane_status = pci_read_word(arg.recv->dev->dev, status_addr);
+              u8 step_status = GET_REG_MASK(lane_status, LMR_PLD_MARGIN_STS);
+              if (!(GET_REG_MASK(lane_status, LMR_CMD_TYPE) == marg_type
+                    && GET_REG_MASK(lane_status, LMR_CMD_RECVN) == arg.recv->recvn
+                    && step_status == 2
+                    && GET_REG_MASK(lane_status, LMR_PLD_ERR_CNT) <= arg.recv->error_limit
+                    && margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND)))
+                {
+                  alive_lanes--;
+                  failed_lanes[i] = true;
+                  arg.results[i].steps[arg.dir] = steps_done - 1;
+                  arg.results[i].statuses[arg.dir]
+                    = (step_status == 3 || step_status == 1 ? MARGIN_NAK : MARGIN_LIM);
+                }
+            }
+        }
+
+      arg.steps_lane_done = steps_done;
+      margin_log_margining(arg);
+    }
+
+  for (int i = 0; i < arg.lanes_n; i++)
+    {
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, CLEAR_ERROR_LOG(arg.recv->recvn));
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, GO_TO_NORMAL_SETTINGS(arg.recv->recvn));
+      margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
+    }
+}
+
+/* Awaits that Receiver is prepared through prep_dev function */
+static bool
+margin_test_receiver(struct margin_dev *dev, u8 recvn, struct margin_args *args,
+                     struct margin_results *results)
+{
+  u8 *lanes_to_margin = args->lanes;
+  u8 lanes_n = args->lanes_n;
+
+  struct margin_params params;
+  struct margin_recv recv = { .dev = dev,
+                              .recvn = recvn,
+                              .lane_reversal = false,
+                              .params = &params,
+                              .parallel_lanes = args->parallel_lanes ? args->parallel_lanes : 1,
+                              .error_limit = args->error_limit };
+
+  results->recvn = recvn;
+  results->lanes_n = lanes_n;
+  margin_log_recvn(&recv);
+
+  if (!margin_check_ready_bit(dev->dev))
+    {
+      margin_log("\nMargining Ready bit is Clear.\n");
+      results->test_status = MARGIN_TEST_READY_BIT;
+      return false;
+    }
+
+  if (!read_params_internal(dev, recvn, recv.lane_reversal, &params))
+    {
+      recv.lane_reversal = true;
+      if (!read_params_internal(dev, recvn, recv.lane_reversal, &params))
+        {
+          margin_log("\nError during caps reading.\n");
+          results->test_status = MARGIN_TEST_CAPS;
+          return false;
+        }
+    }
+
+  results->params = params;
+
+  if (recv.parallel_lanes > params.max_lanes + 1)
+    recv.parallel_lanes = params.max_lanes + 1;
+  margin_apply_hw_quirks(&recv);
+  margin_log_hw_quirks(&recv);
+
+  results->tim_off_reported = params.timing_offset != 0;
+  results->volt_off_reported = params.volt_offset != 0;
+  double tim_offset = results->tim_off_reported ? (double)params.timing_offset : 50.0;
+  double volt_offset = results->volt_off_reported ? (double)params.volt_offset : 50.0;
+
+  results->tim_coef = tim_offset / (double)params.timing_steps;
+  results->volt_coef = volt_offset / (double)params.volt_steps * 10.0;
+
+  results->lane_reversal = recv.lane_reversal;
+  results->link_speed = dev->link_speed;
+  results->test_status = MARGIN_TEST_OK;
+
+  margin_log_receiver(&recv);
+
+  results->lanes = xmalloc(sizeof(struct margin_res_lane) * lanes_n);
+  for (int i = 0; i < lanes_n; i++)
+    {
+      results->lanes[i].lane
+        = recv.lane_reversal ? dev->width - lanes_to_margin[i] - 1 : lanes_to_margin[i];
+    }
+
+  if (args->run_margin)
+    {
+      if (args->verbosity > 0)
+        margin_log("\n");
+      struct margin_lanes_data lanes_data
+        = { .recv = &recv, .verbosity = args->verbosity, .steps_utility = args->steps_utility };
+
+      enum margin_dir dir[] = { TIM_LEFT, TIM_RIGHT, VOLT_UP, VOLT_DOWN };
+
+      u8 lanes_done = 0;
+      u8 use_lanes = 0;
+      u8 steps_t = args->steps_t ? args->steps_t : params.timing_steps;
+      u8 steps_v = args->steps_v ? args->steps_v : params.volt_steps;
+
+      while (lanes_done != lanes_n)
+        {
+          use_lanes = (lanes_done + recv.parallel_lanes > lanes_n) ? lanes_n - lanes_done :
+                                                                     recv.parallel_lanes;
+          lanes_data.lanes_numbers = lanes_to_margin + lanes_done;
+          lanes_data.lanes_n = use_lanes;
+          lanes_data.results = results->lanes + lanes_done;
+
+          for (int i = 0; i < 4; i++)
+            {
+              bool timing = dir[i] == TIM_LEFT || dir[i] == TIM_RIGHT;
+              if (!timing && !params.volt_support)
+                continue;
+              if (dir[i] == TIM_RIGHT && !params.ind_left_right_tim)
+                continue;
+              if (dir[i] == VOLT_DOWN && !params.ind_up_down_volt)
+                continue;
+
+              lanes_data.ind = timing ? params.ind_left_right_tim : params.ind_up_down_volt;
+              lanes_data.dir = dir[i];
+              lanes_data.steps_lane_total = timing ? steps_t : steps_v;
+              if (*args->steps_utility >= lanes_data.steps_lane_total)
+                *args->steps_utility -= lanes_data.steps_lane_total;
+              else
+                *args->steps_utility = 0;
+              margin_test_lanes(lanes_data);
+            }
+          lanes_done += use_lanes;
+        }
+      if (args->verbosity > 0)
+        margin_log("\n");
+      if (recv.lane_reversal)
+        {
+          for (int i = 0; i < lanes_n; i++)
+            results->lanes[i].lane = lanes_to_margin[i];
+        }
+    }
+
+  return true;
+}
+
+bool
+margin_read_params(struct pci_access *pacc, struct pci_dev *dev, u8 recvn,
+                   struct margin_params *params)
+{
+  struct pci_cap *cap = pci_find_cap(dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
+  if (!cap)
+    return false;
+  u8 dev_dir = GET_REG_MASK(pci_read_word(dev, cap->addr + PCI_EXP_FLAGS), PCI_EXP_FLAGS_TYPE);
+
+  bool dev_down;
+  if (dev_dir == PCI_EXP_TYPE_ROOT_PORT || dev_dir == PCI_EXP_TYPE_DOWNSTREAM)
+    dev_down = true;
+  else
+    dev_down = false;
+
+  if (recvn == 0)
+    {
+      if (dev_down)
+        recvn = 1;
+      else
+        recvn = 6;
+    }
+
+  if (recvn > 6)
+    return false;
+  if (dev_down && recvn == 6)
+    return false;
+  if (!dev_down && recvn != 6)
+    return false;
+
+  struct pci_dev *down = NULL;
+  struct pci_dev *up = NULL;
+  struct margin_link link;
+
+  for (struct pci_dev *p = pacc->devices; p; p = p->next)
+    {
+      if (dev_down && pci_read_byte(dev, PCI_SECONDARY_BUS) == p->bus && dev->domain == p->domain
+          && p->func == 0)
+        {
+          down = dev;
+          up = p;
+          break;
+        }
+      else if (!dev_down && pci_read_byte(p, PCI_SECONDARY_BUS) == dev->bus
+               && dev->domain == p->domain)
+        {
+          down = p;
+          up = dev;
+          break;
+        }
+    }
+
+  if (!down)
+    return false;
+
+  if (!margin_fill_link(down, up, &link))
+    return false;
+
+  struct margin_dev *dut = (dev_down ? &link.down_port : &link.up_port);
+  if (!margin_check_ready_bit(dut->dev))
+    return false;
+
+  if (!margin_prep_link(&link))
+    return false;
+
+  bool status;
+  bool lane_reversal = false;
+  status = read_params_internal(dut, recvn, lane_reversal, params);
+  if (!status)
+    {
+      lane_reversal = true;
+      status = read_params_internal(dut, recvn, lane_reversal, params);
+    }
+
+  margin_restore_link(&link);
+
+  return status;
+}
+
+enum margin_test_status
+margin_process_args(struct margin_dev *dev, struct margin_args *args)
+{
+  u8 receivers_n = 2 + 2 * dev->retimers_n;
+
+  if (!args->recvs_n)
+    {
+      for (int i = 1; i < receivers_n; i++)
+        args->recvs[i - 1] = i;
+      args->recvs[receivers_n - 1] = 6;
+      args->recvs_n = receivers_n;
+    }
+  else
+    {
+      for (int i = 0; i < args->recvs_n; i++)
+        {
+          u8 recvn = args->recvs[i];
+          if (recvn < 1 || recvn > 6 || (recvn != 6 && recvn > receivers_n - 1))
+            {
+              return MARGIN_TEST_ARGS_RECVS;
+            }
+        }
+    }
+
+  if (!args->lanes_n)
+    {
+      args->lanes_n = dev->width;
+      for (int i = 0; i < args->lanes_n; i++)
+        args->lanes[i] = i;
+    }
+  else
+    {
+      for (int i = 0; i < args->lanes_n; i++)
+        {
+          if (args->lanes[i] >= dev->width)
+            {
+              return MARGIN_TEST_ARGS_LANES;
+            }
+        }
+    }
+
+  return MARGIN_TEST_OK;
+}
+
+struct margin_results *
+margin_test_link(struct margin_link *link, struct margin_args *args, u8 *recvs_n)
+{
+  bool status = margin_prep_link(link);
+
+  u8 receivers_n = status ? args->recvs_n : 1;
+  u8 *receivers = args->recvs;
+
+  margin_log_link(link);
+
+  struct margin_results *results = xmalloc(sizeof(*results) * receivers_n);
+
+  if (!status)
+    {
+      results[0].test_status = MARGIN_TEST_ASPM;
+      margin_log("\nCouldn't disable ASPM on the given Link.\n");
+    }
+
+  if (status)
+    {
+      struct margin_dev *dut;
+      for (int i = 0; i < receivers_n; i++)
+        {
+          dut = receivers[i] == 6 ? &link->up_port : &link->down_port;
+          margin_test_receiver(dut, receivers[i], args, &results[i]);
+        }
+
+      margin_restore_link(link);
+    }
+
+  *recvs_n = receivers_n;
+  return results;
+}
+
+void
+margin_free_results(struct margin_results *results, u8 results_n)
+{
+  for (int i = 0; i < results_n; i++)
+    {
+      if (results[i].test_status == MARGIN_TEST_OK)
+        free(results[i].lanes);
+    }
+  free(results);
+}
diff --git a/lmr/margin_hw.c b/lmr/margin_hw.c
new file mode 100644
index 0000000..fc427c8
--- /dev/null
+++ b/lmr/margin_hw.c
@@ -0,0 +1,160 @@
+/*
+ *	The PCI Utilities -- Verify and prepare devices before margining
+ *
+ *	Copyright (c) 2023 KNS Group LLC (YADRO)
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL v2+.
+ *
+ *	SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "lmr.h"
+
+static u16 special_hw[][4] =
+  // Vendor ID, Device ID, Revision ID, margin_hw
+  { { 0x8086, 0x347A, 0x4, MARGIN_ICE_LAKE_RC }, 
+    { 0xFFFF, 0, 0, MARGIN_HW_DEFAULT } 
+  };
+
+static enum margin_hw
+detect_unique_hw(struct pci_dev *dev)
+{
+  u16 vendor = pci_read_word(dev, PCI_VENDOR_ID);
+  u16 device = pci_read_word(dev, PCI_DEVICE_ID);
+  u8 revision = pci_read_byte(dev, PCI_REVISION_ID);
+
+  for (int i = 0; special_hw[i][0] != 0xFFFF; i++)
+    {
+      if (vendor == special_hw[i][0] && device == special_hw[i][1] && revision == special_hw[i][2])
+        return special_hw[i][3];
+    }
+  return MARGIN_HW_DEFAULT;
+}
+
+bool
+margin_verify_link(struct pci_dev *down_port, struct pci_dev *up_port)
+{
+  struct pci_cap *cap = pci_find_cap(down_port, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
+  if (!cap)
+    return false;
+  if ((pci_read_word(down_port, cap->addr + PCI_EXP_LNKSTA) & PCI_EXP_LNKSTA_SPEED) < 4)
+    return false;
+  if ((pci_read_word(down_port, cap->addr + PCI_EXP_LNKSTA) & PCI_EXP_LNKSTA_SPEED) > 5)
+    return false;
+
+  u8 down_type = pci_read_byte(down_port, PCI_HEADER_TYPE) & 0x7F;
+  u8 down_sec = pci_read_byte(down_port, PCI_SECONDARY_BUS);
+  u8 down_dir
+    = GET_REG_MASK(pci_read_word(down_port, cap->addr + PCI_EXP_FLAGS), PCI_EXP_FLAGS_TYPE);
+
+  // Verify that devices are linked, down_port is Root Port or Downstream Port of Switch,
+  // up_port is Function 0 of a Device
+  if (!(down_sec == up_port->bus && down_type == PCI_HEADER_TYPE_BRIDGE
+        && (down_dir == PCI_EXP_TYPE_ROOT_PORT || down_dir == PCI_EXP_TYPE_DOWNSTREAM)
+        && up_port->func == 0))
+    return false;
+
+  struct pci_cap *pm = pci_find_cap(up_port, PCI_CAP_ID_PM, PCI_CAP_NORMAL);
+  return pm && !(pci_read_word(up_port, pm->addr + PCI_PM_CTRL) & PCI_PM_CTRL_STATE_MASK); // D0
+}
+
+bool
+margin_check_ready_bit(struct pci_dev *dev)
+{
+  struct pci_cap *lmr = pci_find_cap(dev, PCI_EXT_CAP_ID_LMR, PCI_CAP_EXTENDED);
+  return lmr && (pci_read_word(dev, lmr->addr + PCI_LMR_PORT_STS) & PCI_LMR_PORT_STS_READY);
+}
+
+/* Awaits device at 16 GT/s or higher */
+static struct margin_dev
+fill_dev_wrapper(struct pci_dev *dev)
+{
+  struct pci_cap *cap = pci_find_cap(dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
+  struct margin_dev res
+    = { .dev = dev,
+        .lmr_cap_addr = pci_find_cap(dev, PCI_EXT_CAP_ID_LMR, PCI_CAP_EXTENDED)->addr,
+        .width = GET_REG_MASK(pci_read_word(dev, cap->addr + PCI_EXP_LNKSTA), PCI_EXP_LNKSTA_WIDTH),
+        .retimers_n
+        = (!!(pci_read_word(dev, cap->addr + PCI_EXP_LNKSTA2) & PCI_EXP_LINKSTA2_RETIMER))
+          + (!!(pci_read_word(dev, cap->addr + PCI_EXP_LNKSTA2) & PCI_EXP_LINKSTA2_2RETIMERS)),
+        .link_speed = (pci_read_word(dev, cap->addr + PCI_EXP_LNKSTA) & PCI_EXP_LNKSTA_SPEED),
+        .hw = detect_unique_hw(dev) };
+  return res;
+}
+
+bool
+margin_fill_link(struct pci_dev *down_port, struct pci_dev *up_port, struct margin_link *wrappers)
+{
+  if (!margin_verify_link(down_port, up_port))
+    return false;
+  wrappers->down_port = fill_dev_wrapper(down_port);
+  wrappers->up_port = fill_dev_wrapper(up_port);
+  return true;
+}
+
+/* Disable ASPM, set Hardware Autonomous Speed/Width Disable bits */
+static bool
+margin_prep_dev(struct margin_dev *dev)
+{
+  struct pci_cap *pcie = pci_find_cap(dev->dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
+  if (!pcie)
+    return false;
+
+  u16 lnk_ctl = pci_read_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL);
+  dev->aspm = lnk_ctl & PCI_EXP_LNKCTL_ASPM;
+  dev->hawd = !!(lnk_ctl & PCI_EXP_LNKCTL_HWAUTWD);
+  lnk_ctl &= ~PCI_EXP_LNKCTL_ASPM;
+  pci_write_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL, lnk_ctl);
+  if (pci_read_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM)
+    return false;
+
+  lnk_ctl |= PCI_EXP_LNKCTL_HWAUTWD;
+  pci_write_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL, lnk_ctl);
+
+  u16 lnk_ctl2 = pci_read_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL2);
+  dev->hasd = !!(lnk_ctl2 & PCI_EXP_LNKCTL2_SPEED_DIS);
+  lnk_ctl2 |= PCI_EXP_LNKCTL2_SPEED_DIS;
+  pci_write_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL2, lnk_ctl2);
+
+  return true;
+}
+
+/* Restore Device ASPM, Hardware Autonomous Speed/Width settings */
+static void
+margin_restore_dev(struct margin_dev *dev)
+{
+  struct pci_cap *pcie = pci_find_cap(dev->dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
+  if (!pcie)
+    return;
+
+  u16 lnk_ctl = pci_read_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL);
+  lnk_ctl = SET_REG_MASK(lnk_ctl, PCI_EXP_LNKCAP_ASPM, dev->aspm);
+  lnk_ctl = SET_REG_MASK(lnk_ctl, PCI_EXP_LNKCTL_HWAUTWD, dev->hawd);
+  pci_write_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL, lnk_ctl);
+
+  u16 lnk_ctl2 = pci_read_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL2);
+  lnk_ctl2 = SET_REG_MASK(lnk_ctl2, PCI_EXP_LNKCTL2_SPEED_DIS, dev->hasd);
+  pci_write_word(dev->dev, pcie->addr + PCI_EXP_LNKCTL2, lnk_ctl2);
+}
+
+bool
+margin_prep_link(struct margin_link *link)
+{
+  if (!link)
+    return false;
+  if (!margin_prep_dev(&link->down_port))
+    return false;
+  if (!margin_prep_dev(&link->up_port))
+    {
+      margin_restore_dev(&link->down_port);
+      return false;
+    }
+  return true;
+}
+
+void
+margin_restore_link(struct margin_link *link)
+{
+  margin_restore_dev(&link->down_port);
+  margin_restore_dev(&link->up_port);
+}
diff --git a/lmr/margin_log.c b/lmr/margin_log.c
new file mode 100644
index 0000000..b3c4bd5
--- /dev/null
+++ b/lmr/margin_log.c
@@ -0,0 +1,158 @@
+/*
+ *	The PCI Utilities -- Log margining process
+ *
+ *	Copyright (c) 2023 KNS Group LLC (YADRO)
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL v2+.
+ *
+ *	SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "lmr.h"
+
+bool margin_global_logging = false;
+bool margin_print_domain = true;
+
+void
+margin_log(char *format, ...)
+{
+  va_list arg;
+  va_start(arg, format);
+  if (margin_global_logging)
+    vprintf(format, arg);
+  va_end(arg);
+}
+
+void
+margin_log_bdfs(struct pci_dev *down, struct pci_dev *up)
+{
+  if (margin_print_domain)
+    margin_log("%x:%x:%x.%x -> %x:%x:%x.%x", down->domain, down->bus, down->dev, down->func,
+               up->domain, up->bus, up->dev, up->func);
+  else
+    margin_log("%x:%x.%x -> %x:%x.%x", down->bus, down->dev, down->func, up->bus, up->dev,
+               up->func);
+}
+
+void
+margin_log_link(struct margin_link *link)
+{
+  margin_log("Link ");
+  margin_log_bdfs(link->down_port.dev, link->up_port.dev);
+  margin_log("\nNegotiated Link Width: %d\n", link->down_port.width);
+  margin_log("Link Speed: %d.0 GT/s = Gen %d\n", (link->down_port.link_speed - 3) * 16,
+             link->down_port.link_speed);
+  margin_log("Available receivers: ");
+  int receivers_n = 2 + 2 * link->down_port.retimers_n;
+  for (int i = 1; i < receivers_n; i++)
+    margin_log("Rx(%X) - %d, ", 10 + i - 1, i);
+  margin_log("Rx(F) - 6\n");
+}
+
+void
+margin_log_params(struct margin_params *params)
+{
+  margin_log("Independent Error Sampler: %d\n", params->ind_error_sampler);
+  margin_log("Sample Reporting Method: %d\n", params->sample_report_method);
+  margin_log("Independent Left and Right Timing Margining: %d\n", params->ind_left_right_tim);
+  margin_log("Voltage Margining Supported: %d\n", params->volt_support);
+  margin_log("Independent Up and Down Voltage Margining: %d\n", params->ind_up_down_volt);
+  margin_log("Number of Timing Steps: %d\n", params->timing_steps);
+  margin_log("Number of Voltage Steps: %d\n", params->volt_steps);
+  margin_log("Max Timing Offset: %d\n", params->timing_offset);
+  margin_log("Max Voltage Offset: %d\n", params->volt_offset);
+  margin_log("Max Lanes: %d\n", params->max_lanes);
+}
+
+void
+margin_log_recvn(struct margin_recv *recv)
+{
+  margin_log("\nReceiver = Rx(%X)\n", 10 + recv->recvn - 1);
+}
+
+void
+margin_log_receiver(struct margin_recv *recv)
+{
+  margin_log("\nError Count Limit = %d\n", recv->error_limit);
+  margin_log("Parallel Lanes: %d\n\n", recv->parallel_lanes);
+
+  margin_log_params(recv->params);
+
+  if (recv->lane_reversal)
+    {
+      margin_log("\nWarning: device uses Lane Reversal.\n");
+      margin_log("However, utility uses logical lane numbers in arguments and for logging.\n");
+    }
+
+  if (recv->params->timing_offset == 0)
+    margin_log("\nWarning: Vendor chose not to report the Max Timing Offset.\n"
+               "Utility will use its max possible value - 50 (50%% UI).\n");
+  if (recv->params->volt_support && recv->params->volt_offset == 0)
+    margin_log("\nWarning: Vendor chose not to report the Max Voltage Offset.\n"
+               "Utility will use its max possible value - 50 (500 mV).\n");
+}
+
+void
+margin_log_margining(struct margin_lanes_data arg)
+{
+  char *ind_dirs[] = { "Up", "Down", "Left", "Right" };
+  char *non_ind_dirs[] = { "Voltage", "", "Timing" };
+
+  if (arg.verbosity > 0)
+    {
+      margin_log("\033[2K\rMargining - ");
+      if (arg.ind)
+        margin_log("%s", ind_dirs[arg.dir]);
+      else
+        margin_log("%s", non_ind_dirs[arg.dir]);
+
+      u8 lanes_counter = 0;
+      margin_log(" - Lanes ");
+      margin_log("[%d", arg.lanes_numbers[0]);
+      for (int i = 1; i < arg.lanes_n; i++)
+        {
+          if (arg.lanes_numbers[i] - 1 == arg.lanes_numbers[i - 1])
+            {
+              lanes_counter++;
+              if (lanes_counter == 1)
+                margin_log("-");
+              if (i + 1 == arg.lanes_n)
+                margin_log("%d", arg.lanes_numbers[i]);
+            }
+          else
+            {
+              if (lanes_counter > 0)
+                margin_log("%d", arg.lanes_numbers[i - 1]);
+              margin_log(",%d", arg.lanes_numbers[i]);
+              lanes_counter = 0;
+            }
+        }
+      margin_log("]");
+
+      u64 lane_eta_s = (arg.steps_lane_total - arg.steps_lane_done) * MARGIN_STEP_MS / 1000;
+      u64 total_eta_s = *arg.steps_utility * MARGIN_STEP_MS / 1000 + lane_eta_s;
+      margin_log(" - ETA: %3ds Steps: %3d Total ETA: %3dm %2ds", lane_eta_s, arg.steps_lane_done,
+                 total_eta_s / 60, total_eta_s % 60);
+
+      fflush(stdout);
+    }
+}
+
+void
+margin_log_hw_quirks(struct margin_recv *recv)
+{
+  switch (recv->dev->hw)
+    {
+      case MARGIN_ICE_LAKE_RC:
+        if (recv->recvn == 1)
+          margin_log("\nRx(A) is Intel Ice Lake RC port.\n"
+                     "Applying next quirks for margining process:\n"
+                     "  - Set MaxVoltageOffset to 12 (120 mV).\n");
+        break;
+      default:
+        break;
+    }
+}
diff --git a/lmr/margin_results.c b/lmr/margin_results.c
new file mode 100644
index 0000000..4d28f04
--- /dev/null
+++ b/lmr/margin_results.c
@@ -0,0 +1,283 @@
+/*
+ *	The PCI Utilities -- Display/save margining results
+ *
+ *	Copyright (c) 2023 KNS Group LLC (YADRO)
+ *
+ *	Can be freely distributed and used under the terms of the GNU GPL v2+.
+ *
+ *	SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "lmr.h"
+
+enum lane_rating {
+  BAD = 0,
+  OKAY,
+  PERFECT,
+  WEIRD,
+  INIT,
+};
+
+static char *const grades[] = { "Bad", "Okay", "Perfect", "Weird" };
+static char *const sts_strings[] = { "NAK", "LIM", "THR" };
+static const double ui[] = { 62.5 / 100, 31.25 / 100 };
+
+static enum lane_rating
+rate_lane(double value, double min, double recommended, enum lane_rating cur_rate)
+{
+  enum lane_rating res = PERFECT;
+  if (value < recommended)
+    res = OKAY;
+  if (value < min)
+    res = BAD;
+  if (cur_rate == INIT)
+    return res;
+  if (res < cur_rate)
+    return res;
+  else
+    return cur_rate;
+}
+
+static bool
+check_recv_weird(struct margin_results *results, double tim_min, double volt_min)
+{
+  bool result = true;
+
+  struct margin_res_lane *lane;
+  for (int i = 0; i < results->lanes_n && result; i++)
+    {
+      lane = &(results->lanes[i]);
+      if (lane->steps[TIM_LEFT] * results->tim_coef != tim_min)
+        result = false;
+      if (results->params.ind_left_right_tim
+          && lane->steps[TIM_RIGHT] * results->tim_coef != tim_min)
+        result = false;
+      if (results->params.volt_support)
+        {
+          if (lane->steps[VOLT_UP] * results->volt_coef != volt_min)
+            result = false;
+          if (results->params.ind_up_down_volt
+              && lane->steps[VOLT_DOWN] * results->volt_coef != volt_min)
+            result = false;
+        }
+    }
+  return result;
+}
+
+void
+margin_results_print_brief(struct margin_results *results, u8 recvs_n)
+{
+  struct margin_res_lane *lane;
+  struct margin_results *res;
+  struct margin_params params;
+
+  enum lane_rating lane_rating;
+
+  u8 link_speed;
+
+  char *no_test_msgs[] = { "",
+                           "Margining Ready bit is Clear",
+                           "Error during caps reading",
+                           "Margining prerequisites are not satisfied (16/32 GT/s, D0)",
+                           "Invalid lanes specified with arguments",
+                           "Invalid receivers specified with arguments",
+                           "Couldn't disable ASPM" };
+
+  for (int i = 0; i < recvs_n; i++)
+    {
+      res = &(results[i]);
+      params = res->params;
+      link_speed = res->link_speed - 4;
+
+      if (res->test_status != MARGIN_TEST_OK)
+        {
+          if (res->test_status < MARGIN_TEST_PREREQS)
+            printf("Rx(%X) -", 10 + res->recvn - 1);
+          printf(" Couldn't run test (%s)\n\n", no_test_msgs[res->test_status]);
+          continue;
+        }
+
+      if (res->lane_reversal)
+        printf("Rx(%X) - Lane Reversal\n", 10 + res->recvn - 1);
+
+      if (!res->tim_off_reported)
+        printf("Rx(%X) - Attention: Vendor chose not to report the Max Timing Offset.\n"
+               "Utility used its max possible value (50%% UI) for calculations of %% UI and ps.\n"
+               "Keep in mind that for timing results of this receiver only steps values are "
+               "reliable.\n\n",
+               10 + res->recvn - 1);
+      if (params.volt_support && !res->volt_off_reported)
+        printf("Rx(%X) - Attention: Vendor chose not to report the Max Voltage Offset.\n"
+               "Utility used its max possible value (500 mV) for calculations of mV.\n"
+               "Keep in mind that for voltage results of this receiver only steps values are "
+               "reliable.\n\n",
+               10 + res->recvn - 1);
+
+      if (check_recv_weird(res, MARGIN_TIM_MIN, MARGIN_VOLT_MIN))
+        lane_rating = WEIRD;
+      else
+        lane_rating = INIT;
+
+      for (u8 j = 0; j < res->lanes_n; j++)
+        {
+          lane = &(res->lanes[j]);
+          double left_ui = lane->steps[TIM_LEFT] * res->tim_coef;
+          double right_ui = lane->steps[TIM_RIGHT] * res->tim_coef;
+          double up_volt = lane->steps[VOLT_UP] * res->volt_coef;
+          double down_volt = lane->steps[VOLT_DOWN] * res->volt_coef;
+
+          if (lane_rating != WEIRD)
+            {
+              lane_rating = rate_lane(left_ui, MARGIN_TIM_MIN, MARGIN_TIM_RECOMMEND, INIT);
+              if (params.ind_left_right_tim)
+                lane_rating
+                  = rate_lane(right_ui, MARGIN_TIM_MIN, MARGIN_TIM_RECOMMEND, lane_rating);
+              if (params.volt_support)
+                {
+                  lane_rating = rate_lane(up_volt, MARGIN_VOLT_MIN, MARGIN_VOLT_MIN, lane_rating);
+                  if (params.ind_up_down_volt)
+                    lane_rating
+                      = rate_lane(down_volt, MARGIN_VOLT_MIN, MARGIN_VOLT_MIN, lane_rating);
+                }
+            }
+
+          printf("Rx(%X) Lane %2d - %s\t", 10 + res->recvn - 1, lane->lane, grades[lane_rating]);
+          if (params.ind_left_right_tim)
+            printf("L %4.1f%% UI - %5.2fps - %2dst %s, R %4.1f%% UI - %5.2fps - %2dst %s", left_ui,
+                   left_ui * ui[link_speed], lane->steps[TIM_LEFT],
+                   sts_strings[lane->statuses[TIM_LEFT]], right_ui, right_ui * ui[link_speed],
+                   lane->steps[TIM_RIGHT], sts_strings[lane->statuses[TIM_RIGHT]]);
+          else
+            printf("T %4.1f%% UI - %5.2fps - %2dst %s", left_ui, left_ui * ui[link_speed],
+                   lane->steps[TIM_LEFT], sts_strings[lane->statuses[TIM_LEFT]]);
+          if (params.volt_support)
+            {
+              if (params.ind_up_down_volt)
+                printf(", U %5.1f mV - %3dst %s, D %5.1f mV - %3dst %s", up_volt,
+                       lane->steps[VOLT_UP], sts_strings[lane->statuses[VOLT_UP]], down_volt,
+                       lane->steps[VOLT_DOWN], sts_strings[lane->statuses[VOLT_DOWN]]);
+              else
+                printf(", V %5.1f mV - %3dst %s", up_volt, lane->steps[VOLT_UP],
+                       sts_strings[lane->statuses[VOLT_UP]]);
+            }
+          printf("\n");
+        }
+      printf("\n");
+    }
+}
+
+void
+margin_results_save_csv(struct margin_results *results, u8 recvs_n, char *dir,
+                        struct pci_dev *up_port)
+{
+  char timestamp[64];
+  time_t tim = time(NULL);
+  strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", gmtime(&tim));
+
+  size_t pathlen = strlen(dir) + 128;
+  char *path = xmalloc(pathlen);
+  FILE *csv;
+
+  struct margin_res_lane *lane;
+  struct margin_results *res;
+  struct margin_params params;
+
+  enum lane_rating lane_rating;
+  u8 link_speed;
+
+  for (int i = 0; i < recvs_n; i++)
+    {
+      res = &(results[i]);
+      params = res->params;
+      link_speed = res->link_speed - 4;
+
+      if (res->test_status != MARGIN_TEST_OK)
+        continue;
+      snprintf(path, pathlen, "%s/lmr_%0*x.%02x.%02x.%x_Rx%X_%s.csv", dir,
+               up_port->domain_16 == 0xffff ? 8 : 4, up_port->domain, up_port->bus, up_port->dev,
+               up_port->func, 10 + res->recvn - 1, timestamp);
+      csv = fopen(path, "w");
+      if (!csv)
+        die("Error while saving %s\n", path);
+
+      fprintf(csv, "Lane,Lane Status,Left %% UI,Left ps,Left Steps,Left Status,"
+                   "Right %% UI,Right ps,Right Steps,Right Status,"
+                   "Time %% UI,Time ps,Time Steps,Time Status,"
+                   "Up mV,Up Steps,Up Status,Down mV,Down Steps,Down Status,"
+                   "Voltage mV,Voltage Steps,Voltage Status\n");
+
+      if (check_recv_weird(res, MARGIN_TIM_MIN, MARGIN_VOLT_MIN))
+        lane_rating = WEIRD;
+      else
+        lane_rating = INIT;
+
+      for (int j = 0; j < res->lanes_n; j++)
+        {
+          lane = &(res->lanes[j]);
+          double left_ui = lane->steps[TIM_LEFT] * res->tim_coef;
+          double right_ui = lane->steps[TIM_RIGHT] * res->tim_coef;
+          double up_volt = lane->steps[VOLT_UP] * res->volt_coef;
+          double down_volt = lane->steps[VOLT_DOWN] * res->volt_coef;
+
+          if (lane_rating != WEIRD)
+            {
+              lane_rating = rate_lane(left_ui, MARGIN_TIM_MIN, MARGIN_TIM_RECOMMEND, INIT);
+              if (params.ind_left_right_tim)
+                lane_rating
+                  = rate_lane(right_ui, MARGIN_TIM_MIN, MARGIN_TIM_RECOMMEND, lane_rating);
+              if (params.volt_support)
+                {
+                  lane_rating = rate_lane(up_volt, MARGIN_VOLT_MIN, MARGIN_VOLT_MIN, lane_rating);
+                  if (params.ind_up_down_volt)
+                    lane_rating
+                      = rate_lane(down_volt, MARGIN_VOLT_MIN, MARGIN_VOLT_MIN, lane_rating);
+                }
+            }
+
+          fprintf(csv, "%d,%s,", lane->lane, grades[lane_rating]);
+          if (params.ind_left_right_tim)
+            {
+              fprintf(csv, "%f,%f,%d,%s,%f,%f,%d,%s,NA,NA,NA,NA,", left_ui,
+                      left_ui * ui[link_speed], lane->steps[TIM_LEFT],
+                      sts_strings[lane->statuses[TIM_LEFT]], right_ui, right_ui * ui[link_speed],
+                      lane->steps[TIM_RIGHT], sts_strings[lane->statuses[TIM_RIGHT]]);
+            }
+          else
+            {
+              for (int k = 0; k < 8; k++)
+                fprintf(csv, "NA,");
+              fprintf(csv, "%f,%f,%d,%s,", left_ui, left_ui * ui[link_speed], lane->steps[TIM_LEFT],
+                      sts_strings[lane->statuses[TIM_LEFT]]);
+            }
+          if (params.volt_support)
+            {
+              if (params.ind_up_down_volt)
+                {
+                  fprintf(csv, "%f,%d,%s,%f,%d,%s,NA,NA,NA\n", up_volt, lane->steps[VOLT_UP],
+                          sts_strings[lane->statuses[VOLT_UP]], down_volt, lane->steps[VOLT_DOWN],
+                          sts_strings[lane->statuses[VOLT_DOWN]]);
+                }
+              else
+                {
+                  for (int k = 0; k < 6; k++)
+                    fprintf(csv, "NA,");
+                  fprintf(csv, "%f,%d,%s\n", up_volt, lane->steps[VOLT_UP],
+                          sts_strings[lane->statuses[VOLT_UP]]);
+                }
+            }
+          else
+            {
+              for (int k = 0; k < 8; k++)
+                fprintf(csv, "NA,");
+              fprintf(csv, "NA\n");
+            }
+        }
+      fclose(csv);
+    }
+  free(path);
+}