path: root/src/test/osd/test_scrub_sched.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

/// \file testing the scrub scheduling algorithm

#include <gtest/gtest.h>

#include <algorithm>
#include <map>

#include "common/async/context_pool.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "global/global_init.h"
#include "include/utime_fmt.h"
#include "mon/MonClient.h"
#include "msg/Messenger.h"
#include "os/ObjectStore.h"
#include "osd/PG.h"
#include "osd/osd_types.h"
#include "osd/osd_types_fmt.h"
#include "osd/scrubber/osd_scrub_sched.h"
#include "osd/scrubber_common.h"

int main(int argc, char** argv)
{
  std::map<std::string, std::string> defaults = {
    // make sure we have 3 copies, or some tests won't work
    {"osd_pool_default_size", "3"},
    // our map is flat, so just try and split across OSDs, not hosts or whatever
    {"osd_crush_chooseleaf_type", "0"},
  };
  std::vector<const char*> args(argv, argv + argc);
  auto cct = global_init(&defaults,
			 args,
			 CEPH_ENTITY_TYPE_CLIENT,
			 CODE_ENVIRONMENT_UTILITY,
			 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
  common_init_finish(g_ceph_context);
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}

using schedule_result_t = Scrub::schedule_result_t;
using ScrubJobRef = ScrubQueue::ScrubJobRef;
using qu_state_t = ScrubQueue::qu_state_t;

/// enabling access into ScrubQueue internals
class ScrubSchedTestWrapper : public ScrubQueue {
 public:
  ScrubSchedTestWrapper(Scrub::ScrubSchedListener& osds)
      : ScrubQueue(g_ceph_context, osds)
  {}

  void rm_unregistered_jobs()
  {
    ScrubQueue::rm_unregistered_jobs(to_scrub);
    ScrubQueue::rm_unregistered_jobs(penalized);
  }

  ScrubQContainer collect_ripe_jobs()
  {
    return ScrubQueue::collect_ripe_jobs(to_scrub, time_now());
  }

  /**
   * unit-test support for faking the current time. When
   * not activated specifically - the default is to use ceph_clock_now()
   */
  void set_time_for_testing(long faked_now)
  {
    m_time_for_testing = utime_t{timeval{faked_now}};
  }
  void clear_time_for_testing() { m_time_for_testing.reset(); }
  mutable std::optional<utime_t> m_time_for_testing;

  utime_t time_now() const final
  {
    if (m_time_for_testing) {
      m_time_for_testing->tv.tv_nsec += 1'000'000;
    }
    return m_time_for_testing.value_or(ceph_clock_now());
  }

  ~ScrubSchedTestWrapper() override = default;
};


/**
 * providing the small number of OSD services used when scheduling
 * a scrub
 */
class FakeOsd : public Scrub::ScrubSchedListener {
 public:
  FakeOsd(int osd_num) : m_osd_num(osd_num) {}

  int get_nodeid() const final { return m_osd_num; }

  schedule_result_t initiate_a_scrub(spg_t pgid,
				     bool allow_requested_repair_only) final
  {
    std::ignore = allow_requested_repair_only;
    auto res = m_next_response.find(pgid);
    if (res == m_next_response.end()) {
      return schedule_result_t::no_such_pg;
    }
    return m_next_response[pgid];
  }

  void set_initiation_response(spg_t pgid, schedule_result_t result)
  {
    m_next_response[pgid] = result;
  }

 private:
  int m_osd_num;
  std::map<spg_t, schedule_result_t> m_next_response;
};


/// the static blueprint for creating a scrub job in the scrub queue
struct sjob_config_t {
  spg_t spg;
  bool are_stats_valid;

  utime_t history_scrub_stamp;
  std::optional<double> pool_conf_min;
  std::optional<double> pool_conf_max;
  bool is_must;
  bool is_need_auto;
  ScrubQueue::scrub_schedule_t initial_schedule;
};


/**
 * the runtime configuration for a scrub job. Created basde on the blueprint
 * above (sjob_config_t)
 */
struct sjob_dynamic_data_t {
  sjob_config_t initial_config;
  pg_info_t mocked_pg_info;
  pool_opts_t mocked_pool_opts;
  requested_scrub_t request_flags;
  ScrubQueue::ScrubJobRef job;
};

class TestScrubSched : public ::testing::Test {
 public:
  TestScrubSched() = default;

 protected:
  int m_osd_num{1};
  FakeOsd m_osds{m_osd_num};
  std::unique_ptr<ScrubSchedTestWrapper> m_sched{
    new ScrubSchedTestWrapper(m_osds)};

  /// the pg-info is queried for stats validity and for the last-scrub-stamp
  pg_info_t pg_info{};

  /// the pool configuration holds some per-pool scrub timing settings
  pool_opts_t pool_opts{};

  /**
   * the scrub-jobs created for the tests, along with their corresponding
   * "pg info" and pool configuration. In real life - the scrub jobs
   * are owned by the respective PGs.
   */
  std::vector<sjob_dynamic_data_t> m_scrub_jobs;

 protected:
  sjob_dynamic_data_t create_scrub_job(const sjob_config_t& sjob_data)
  {
    sjob_dynamic_data_t dyn_data;
    dyn_data.initial_config = sjob_data;

    // populate the 'pool options' object with the scrub timing settings
    if (sjob_data.pool_conf_min) {
      dyn_data.mocked_pool_opts.set<double>(pool_opts_t::SCRUB_MIN_INTERVAL,
					    sjob_data.pool_conf_min.value());
    }
    if (sjob_data.pool_conf_max) {
      dyn_data.mocked_pool_opts.set(pool_opts_t::SCRUB_MAX_INTERVAL,
				    sjob_data.pool_conf_max.value());
    }

    // create the 'pg info' object with the stats
    dyn_data.mocked_pg_info = pg_info_t{sjob_data.spg};

    dyn_data.mocked_pg_info.history.last_scrub_stamp =
      sjob_data.history_scrub_stamp;
    dyn_data.mocked_pg_info.stats.stats_invalid = !sjob_data.are_stats_valid;

    // fake hust the required 'requested-scrub' flags
    std::cout << "request_flags: sjob_data.is_must " << sjob_data.is_must
	      << std::endl;
    dyn_data.request_flags.must_scrub = sjob_data.is_must;
    dyn_data.request_flags.need_auto = sjob_data.is_need_auto;

    // create the scrub job
    dyn_data.job = ceph::make_ref<ScrubQueue::ScrubJob>(g_ceph_context,
							sjob_data.spg,
							m_osd_num);
    m_scrub_jobs.push_back(dyn_data);
    return dyn_data;
  }

  void register_job_set(const std::vector<sjob_config_t>& job_configs)
  {
    std::for_each(job_configs.begin(),
		  job_configs.end(),
		  [this](const sjob_config_t& sj) {
		    auto dynjob = create_scrub_job(sj);
		    m_sched->register_with_osd(
		      dynjob.job,
		      m_sched->determine_scrub_time(dynjob.request_flags,
						    dynjob.mocked_pg_info,
						    dynjob.mocked_pool_opts));
		  });
  }

  /// count the scrub-jobs that are currently in a specific state
  int count_scrub_jobs_in_state(qu_state_t state)
  {
    return std::count_if(m_scrub_jobs.begin(),
			 m_scrub_jobs.end(),
			 [state](const sjob_dynamic_data_t& sj) {
			   return sj.job->state == state;
			 });
  }

  void list_testers_jobs(std::string hdr)
  {
    std::cout << fmt::format("{}: {} jobs created for the test:",
			     hdr,
			     m_scrub_jobs.size())
	      << std::endl;
    for (const auto& job : m_scrub_jobs) {
      std::cout << fmt::format("\t{}: job {}", hdr, *job.job) << std::endl;
    }
  }

  void print_all_states(std::string hdr)
  {
    std::cout << fmt::format(
		   "{}: Created:{}. Per state: not-reg:{} reg:{} unreg:{}",
		   hdr,
		   m_scrub_jobs.size(),
		   count_scrub_jobs_in_state(qu_state_t::not_registered),
		   count_scrub_jobs_in_state(qu_state_t::registered),
		   count_scrub_jobs_in_state(qu_state_t::unregistering))
	      << std::endl;
  }

  void debug_print_jobs(std::string hdr,
			const ScrubQueue::ScrubQContainer& jobs)
  {
    std::cout << fmt::format("{}: time now {}", hdr, m_sched->time_now())
	      << std::endl;
    for (const auto& job : jobs) {
      std::cout << fmt::format(
		     "\t{}: job {} ({}): scheduled {}",
		     hdr,
		     job->pgid,
		     job->scheduling_state(m_sched->time_now(), false),
		     job->get_sched_time())
		<< std::endl;
    }
  }
};

// ///////////////////////////////////////////////////////////////////////////
// test data. Scrub-job creation requires a PG-id, and a set of 'scrub request'
// flags

namespace {

// the times used during the tests are offset to 1.1.2000, so that
// utime_t formatting will treat them as absolute (not as a relative time)
static const auto epoch_2000 = 946'684'800;

std::vector<sjob_config_t> sjob_configs = {
  {
    spg_t{pg_t{1, 1}},
    true,					      // PG has valid stats
    utime_t{std::time_t(epoch_2000 + 1'000'000), 0},  // last-scrub-stamp
    100.0,			    // min scrub delay in pool config
    std::nullopt,		    // max scrub delay in pool config
    false,			    // must-scrub
    false,			    // need-auto
    ScrubQueue::scrub_schedule_t{}  // initial schedule
  },

  {spg_t{pg_t{4, 1}},
   true,
   utime_t{epoch_2000 + 1'000'000, 0},
   100.0,
   std::nullopt,
   true,
   false,
   ScrubQueue::scrub_schedule_t{}},

  {spg_t{pg_t{7, 1}},
   true,
   utime_t{},
   1.0,
   std::nullopt,
   false,
   false,
   ScrubQueue::scrub_schedule_t{}},

  {spg_t{pg_t{5, 1}},
   true,
   utime_t{epoch_2000 + 1'900'000, 0},
   1.0,
   std::nullopt,
   false,
   false,
   ScrubQueue::scrub_schedule_t{}}};

}  // anonymous namespace

// //////////////////////////// tests ////////////////////////////////////////

/// basic test: scheduling simple jobs, validating their calculated schedule
TEST_F(TestScrubSched, populate_queue)
{
  ASSERT_EQ(0, m_sched->list_registered_jobs().size());

  auto dynjob_0 = create_scrub_job(sjob_configs[0]);
  auto suggested = m_sched->determine_scrub_time(dynjob_0.request_flags,
						 dynjob_0.mocked_pg_info,
						 dynjob_0.mocked_pool_opts);
  m_sched->register_with_osd(dynjob_0.job, suggested);
  std::cout << fmt::format("scheduled at: {}", dynjob_0.job->get_sched_time())
	    << std::endl;

  auto dynjob_1 = create_scrub_job(sjob_configs[1]);
  suggested = m_sched->determine_scrub_time(dynjob_1.request_flags,
					    dynjob_1.mocked_pg_info,
					    dynjob_1.mocked_pool_opts);
  m_sched->register_with_osd(dynjob_1.job, suggested);
  std::cout << fmt::format("scheduled at: {}", dynjob_1.job->get_sched_time())
	    << std::endl;

  EXPECT_EQ(dynjob_1.job->get_sched_time(), utime_t(1, 1));
  EXPECT_EQ(2, m_sched->list_registered_jobs().size());
}

/// validate the states of the scrub-jobs (as set in the jobs themselves)
TEST_F(TestScrubSched, states)
{
  m_sched->set_time_for_testing(epoch_2000);
  register_job_set(sjob_configs);
  list_testers_jobs("testing states");
  EXPECT_EQ(sjob_configs.size(), m_sched->list_registered_jobs().size());

  // check the initial state of the jobs
  print_all_states("<initial state>");
  m_sched->rm_unregistered_jobs();
  EXPECT_EQ(0, count_scrub_jobs_in_state(qu_state_t::not_registered));

  // now - remove a couple of them
  m_sched->remove_from_osd_queue(m_scrub_jobs[2].job);
  m_sched->remove_from_osd_queue(m_scrub_jobs[1].job);
  m_sched->remove_from_osd_queue(m_scrub_jobs[2].job);	// should have no effect

  print_all_states("<w/ 2 jobs removed>");
  EXPECT_EQ(2, count_scrub_jobs_in_state(qu_state_t::registered));
  EXPECT_EQ(2, count_scrub_jobs_in_state(qu_state_t::unregistering));

  m_sched->rm_unregistered_jobs();
  EXPECT_EQ(2, count_scrub_jobs_in_state(qu_state_t::not_registered));
  std::cout << fmt::format("inp size: {}. In list-registered: {}",
			   sjob_configs.size(),
			   m_sched->list_registered_jobs().size())
	    << std::endl;
  EXPECT_EQ(sjob_configs.size() - 2, m_sched->list_registered_jobs().size());
}

/// jobs that are ripe should be in the ready list, sorted by their scheduled
/// time
TEST_F(TestScrubSched, ready_list)
{
  m_sched->set_time_for_testing(epoch_2000 + 900'000);
  register_job_set(sjob_configs);
  list_testers_jobs("testing states");
  EXPECT_EQ(sjob_configs.size(), m_sched->list_registered_jobs().size());

  m_sched->set_time_for_testing(epoch_2000 + 1'000'000);
  auto all_reg_jobs = m_sched->list_registered_jobs();
  debug_print_jobs("registered", all_reg_jobs);

  auto ripe_jobs = m_sched->collect_ripe_jobs();
  EXPECT_EQ(2, ripe_jobs.size());
  debug_print_jobs("ready_list", ripe_jobs);

  m_sched->set_time_for_testing(epoch_2000 + 3'000'000);
  // all jobs should be in the ready list
  ripe_jobs = m_sched->collect_ripe_jobs();
  EXPECT_EQ(4, ripe_jobs.size());
  debug_print_jobs("ready_list", ripe_jobs);
}