diff options
Diffstat (limited to 'qa/tasks/scrub_test.py')
| -rw-r--r-- | qa/tasks/scrub_test.py | 413 |
1 files changed, 413 insertions, 0 deletions
diff --git a/qa/tasks/scrub_test.py b/qa/tasks/scrub_test.py new file mode 100644 index 000000000..edf106952 --- /dev/null +++ b/qa/tasks/scrub_test.py @@ -0,0 +1,413 @@ +"""Scrub testing""" + +import contextlib +import json +import logging +import os +import time +import tempfile + +from tasks import ceph_manager +from teuthology import misc as teuthology + +log = logging.getLogger(__name__) + + +def wait_for_victim_pg(manager, poolid): + """Return a PG with some data and its acting set""" + # wait for some PG to have data that we can mess with + victim = None + while victim is None: + stats = manager.get_pg_stats() + for pg in stats: + pgid = str(pg['pgid']) + pgpool = int(pgid.split('.')[0]) + if poolid != pgpool: + continue + size = pg['stat_sum']['num_bytes'] + if size > 0: + victim = pg['pgid'] + acting = pg['acting'] + return victim, acting + time.sleep(3) + + +def find_victim_object(ctx, pg, osd): + """Return a file to be fuzzed""" + (osd_remote,) = ctx.cluster.only('osd.%d' % osd).remotes.keys() + data_path = os.path.join( + '/var/lib/ceph/osd', + 'ceph-{id}'.format(id=osd), + 'fuse', + '{pg}_head'.format(pg=pg), + 'all', + ) + + # fuzz time + ls_out = osd_remote.sh('sudo ls %s' % data_path) + + # find an object file we can mess with (and not the pg info object) + osdfilename = next(line for line in ls_out.split('\n') + if not line.endswith('::::head#')) + assert osdfilename is not None + + # Get actual object name from osd stored filename + objname = osdfilename.split(':')[4] + return osd_remote, os.path.join(data_path, osdfilename), objname + + +def corrupt_file(osd_remote, path): + # put a single \0 at the beginning of the file + osd_remote.run( + args=['sudo', 'dd', + 'if=/dev/zero', + 'of=%s/data' % path, + 'bs=1', 'count=1', 'conv=notrunc'] + ) + + +def get_pgnum(pgid): + pos = pgid.find('.') + assert pos != -1 + return pgid[pos+1:] + + +def deep_scrub(manager, victim, pool): + # scrub, verify inconsistent + pgnum = get_pgnum(victim) + manager.do_pg_scrub(pool, pgnum, 'deep-scrub') + + stats = manager.get_single_pg_stats(victim) + inconsistent = stats['state'].find('+inconsistent') != -1 + assert inconsistent + + +def repair(manager, victim, pool): + # repair, verify no longer inconsistent + pgnum = get_pgnum(victim) + manager.do_pg_scrub(pool, pgnum, 'repair') + + stats = manager.get_single_pg_stats(victim) + inconsistent = stats['state'].find('+inconsistent') != -1 + assert not inconsistent + + +def test_repair_corrupted_obj(ctx, manager, pg, osd_remote, obj_path, pool): + corrupt_file(osd_remote, obj_path) + deep_scrub(manager, pg, pool) + repair(manager, pg, pool) + + +def test_repair_bad_omap(ctx, manager, pg, osd, objname): + # Test deep-scrub with various omap modifications + # Modify omap on specific osd + log.info('fuzzing omap of %s' % objname) + manager.osd_admin_socket(osd, ['rmomapkey', 'rbd', objname, 'key']) + manager.osd_admin_socket(osd, ['setomapval', 'rbd', objname, + 'badkey', 'badval']) + manager.osd_admin_socket(osd, ['setomapheader', 'rbd', objname, 'badhdr']) + + deep_scrub(manager, pg, 'rbd') + # please note, the repair here is errnomous, it rewrites the correct omap + # digest and data digest on the replicas with the corresponding digests + # from the primary osd which is hosting the victim object, see + # find_victim_object(). + # so we need to either put this test and the end of this task or + # undo the mess-up manually before the "repair()" that just ensures + # the cleanup is sane, otherwise the succeeding tests will fail. if they + # try set "badkey" in hope to get an "inconsistent" pg with a deep-scrub. + manager.osd_admin_socket(osd, ['setomapheader', 'rbd', objname, 'hdr']) + manager.osd_admin_socket(osd, ['rmomapkey', 'rbd', objname, 'badkey']) + manager.osd_admin_socket(osd, ['setomapval', 'rbd', objname, + 'key', 'val']) + repair(manager, pg, 'rbd') + + +class MessUp: + def __init__(self, manager, osd_remote, pool, osd_id, + obj_name, obj_path, omap_key, omap_val): + self.manager = manager + self.osd = osd_remote + self.pool = pool + self.osd_id = osd_id + self.obj = obj_name + self.path = obj_path + self.omap_key = omap_key + self.omap_val = omap_val + + @contextlib.contextmanager + def _test_with_file(self, messup_cmd, *checks): + temp = tempfile.mktemp() + backup_cmd = ['sudo', 'cp', os.path.join(self.path, 'data'), temp] + self.osd.run(args=backup_cmd) + self.osd.run(args=messup_cmd.split()) + yield checks + create_cmd = ['sudo', 'mkdir', self.path] + self.osd.run(args=create_cmd, check_status=False) + restore_cmd = ['sudo', 'cp', temp, os.path.join(self.path, 'data')] + self.osd.run(args=restore_cmd) + + def remove(self): + cmd = 'sudo rmdir {path}'.format(path=self.path) + return self._test_with_file(cmd, 'missing') + + def append(self): + cmd = 'sudo dd if=/dev/zero of={path}/data bs=1 count=1 ' \ + 'conv=notrunc oflag=append'.format(path=self.path) + return self._test_with_file(cmd, + 'data_digest_mismatch', + 'size_mismatch') + + def truncate(self): + cmd = 'sudo dd if=/dev/null of={path}/data'.format(path=self.path) + return self._test_with_file(cmd, + 'data_digest_mismatch', + 'size_mismatch') + + def change_obj(self): + cmd = 'sudo dd if=/dev/zero of={path}/data bs=1 count=1 ' \ + 'conv=notrunc'.format(path=self.path) + return self._test_with_file(cmd, + 'data_digest_mismatch') + + @contextlib.contextmanager + def rm_omap(self): + cmd = ['rmomapkey', self.pool, self.obj, self.omap_key] + self.manager.osd_admin_socket(self.osd_id, cmd) + yield ('omap_digest_mismatch',) + cmd = ['setomapval', self.pool, self.obj, + self.omap_key, self.omap_val] + self.manager.osd_admin_socket(self.osd_id, cmd) + + @contextlib.contextmanager + def add_omap(self): + cmd = ['setomapval', self.pool, self.obj, 'badkey', 'badval'] + self.manager.osd_admin_socket(self.osd_id, cmd) + yield ('omap_digest_mismatch',) + cmd = ['rmomapkey', self.pool, self.obj, 'badkey'] + self.manager.osd_admin_socket(self.osd_id, cmd) + + @contextlib.contextmanager + def change_omap(self): + cmd = ['setomapval', self.pool, self.obj, self.omap_key, 'badval'] + self.manager.osd_admin_socket(self.osd_id, cmd) + yield ('omap_digest_mismatch',) + cmd = ['setomapval', self.pool, self.obj, self.omap_key, self.omap_val] + self.manager.osd_admin_socket(self.osd_id, cmd) + + +class InconsistentObjChecker: + """Check the returned inconsistents/inconsistent info""" + + def __init__(self, osd, acting, obj_name): + self.osd = osd + self.acting = acting + self.obj = obj_name + assert self.osd in self.acting + + def basic_checks(self, inc): + assert inc['object']['name'] == self.obj + assert inc['object']['snap'] == "head" + assert len(inc['shards']) == len(self.acting), \ + "the number of returned shard does not match with the acting set" + + def run(self, check, inc): + func = getattr(self, check) + func(inc) + + def _check_errors(self, inc, err_name): + bad_found = False + good_found = False + for shard in inc['shards']: + log.info('shard = %r' % shard) + log.info('err = %s' % err_name) + assert 'osd' in shard + osd = shard['osd'] + err = err_name in shard['errors'] + if osd == self.osd: + assert bad_found is False, \ + "multiple entries found for the given OSD" + assert err is True, \ + "Didn't find '{err}' in errors".format(err=err_name) + bad_found = True + else: + assert osd in self.acting, "shard not in acting set" + assert err is False, \ + "Expected '{err}' in errors".format(err=err_name) + good_found = True + assert bad_found is True, \ + "Shard for osd.{osd} not found".format(osd=self.osd) + assert good_found is True, \ + "No other acting shards found" + + def _check_attrs(self, inc, attr_name): + bad_attr = None + good_attr = None + for shard in inc['shards']: + log.info('shard = %r' % shard) + log.info('attr = %s' % attr_name) + assert 'osd' in shard + osd = shard['osd'] + attr = shard.get(attr_name, False) + if osd == self.osd: + assert bad_attr is None, \ + "multiple entries found for the given OSD" + bad_attr = attr + else: + assert osd in self.acting, "shard not in acting set" + assert good_attr is None or good_attr == attr, \ + "multiple good attrs found" + good_attr = attr + assert bad_attr is not None, \ + "bad {attr} not found".format(attr=attr_name) + assert good_attr is not None, \ + "good {attr} not found".format(attr=attr_name) + assert good_attr != bad_attr, \ + "bad attr is identical to the good ones: " \ + "{0} == {1}".format(good_attr, bad_attr) + + def data_digest_mismatch(self, inc): + assert 'data_digest_mismatch' in inc['errors'] + self._check_attrs(inc, 'data_digest') + + def missing(self, inc): + assert 'missing' in inc['union_shard_errors'] + self._check_errors(inc, 'missing') + + def size_mismatch(self, inc): + assert 'size_mismatch' in inc['errors'] + self._check_attrs(inc, 'size') + + def omap_digest_mismatch(self, inc): + assert 'omap_digest_mismatch' in inc['errors'] + self._check_attrs(inc, 'omap_digest') + + +def test_list_inconsistent_obj(ctx, manager, osd_remote, pg, acting, osd_id, + obj_name, obj_path): + mon = manager.controller + pool = 'rbd' + omap_key = 'key' + omap_val = 'val' + manager.do_rados(['setomapval', obj_name, omap_key, omap_val], pool=pool) + # Update missing digests, requires "osd deep scrub update digest min age: 0" + pgnum = get_pgnum(pg) + manager.do_pg_scrub(pool, pgnum, 'deep-scrub') + + messup = MessUp(manager, osd_remote, pool, osd_id, obj_name, obj_path, + omap_key, omap_val) + for test in [messup.rm_omap, messup.add_omap, messup.change_omap, + messup.append, messup.truncate, messup.change_obj, + messup.remove]: + with test() as checks: + deep_scrub(manager, pg, pool) + cmd = 'rados list-inconsistent-pg {pool} ' \ + '--format=json'.format(pool=pool) + pgs = json.loads(mon.sh(cmd)) + assert pgs == [pg] + + cmd = 'rados list-inconsistent-obj {pg} ' \ + '--format=json'.format(pg=pg) + objs = json.loads(mon.sh(cmd)) + assert len(objs['inconsistents']) == 1 + + checker = InconsistentObjChecker(osd_id, acting, obj_name) + inc_obj = objs['inconsistents'][0] + log.info('inc = %r', inc_obj) + checker.basic_checks(inc_obj) + for check in checks: + checker.run(check, inc_obj) + + +def task(ctx, config): + """ + Test [deep] scrub + + tasks: + - chef: + - install: + - ceph: + log-ignorelist: + - '!= data_digest' + - '!= omap_digest' + - '!= size' + - deep-scrub 0 missing, 1 inconsistent objects + - deep-scrub [0-9]+ errors + - repair 0 missing, 1 inconsistent objects + - repair [0-9]+ errors, [0-9]+ fixed + - shard [0-9]+ .* : missing + - deep-scrub 1 missing, 1 inconsistent objects + - does not match object info size + - attr name mistmatch + - deep-scrub 1 missing, 0 inconsistent objects + - failed to pick suitable auth object + - candidate size [0-9]+ info size [0-9]+ mismatch + conf: + osd: + osd deep scrub update digest min age: 0 + - scrub_test: + """ + if config is None: + config = {} + assert isinstance(config, dict), \ + 'scrub_test task only accepts a dict for configuration' + first_mon = teuthology.get_first_mon(ctx, config) + (mon,) = ctx.cluster.only(first_mon).remotes.keys() + + num_osds = teuthology.num_instances_of_type(ctx.cluster, 'osd') + log.info('num_osds is %s' % num_osds) + + manager = ceph_manager.CephManager( + mon, + ctx=ctx, + logger=log.getChild('ceph_manager'), + ) + + while len(manager.get_osd_status()['up']) < num_osds: + time.sleep(10) + + for i in range(num_osds): + manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'injectargs', + '--', '--osd-objectstore-fuse') + manager.flush_pg_stats(range(num_osds)) + manager.wait_for_clean() + + osd_dump = manager.get_osd_dump_json() + poolid = -1 + for p in osd_dump['pools']: + if p['pool_name'] == 'rbd': + poolid = p['pool'] + break + assert poolid != -1 + + # write some data + p = manager.do_rados(['bench', '--no-cleanup', '1', 'write', '-b', '4096'], pool='rbd') + log.info('err is %d' % p.exitstatus) + + # wait for some PG to have data that we can mess with + pg, acting = wait_for_victim_pg(manager, poolid) + osd = acting[0] + + osd_remote, obj_path, obj_name = find_victim_object(ctx, pg, osd) + manager.do_rados(['setomapval', obj_name, 'key', 'val'], pool='rbd') + log.info('err is %d' % p.exitstatus) + manager.do_rados(['setomapheader', obj_name, 'hdr'], pool='rbd') + log.info('err is %d' % p.exitstatus) + + # Update missing digests, requires "osd deep scrub update digest min age: 0" + pgnum = get_pgnum(pg) + manager.do_pg_scrub('rbd', pgnum, 'deep-scrub') + + log.info('messing with PG %s on osd %d' % (pg, osd)) + test_repair_corrupted_obj(ctx, manager, pg, osd_remote, obj_path, 'rbd') + test_repair_bad_omap(ctx, manager, pg, osd, obj_name) + test_list_inconsistent_obj(ctx, manager, osd_remote, pg, acting, osd, + obj_name, obj_path) + log.info('test successful!') + + # shut down fuse mount + for i in range(num_osds): + manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'injectargs', + '--', '--no-osd-objectstore-fuse') + time.sleep(5) + log.info('done') |