summaryrefslogtreecommitdiffstats
path: root/qa/tasks/cephfs/test_auto_repair.py
blob: e6f0a8f0bd4cb5d0d5dbdbc7f2796c4a5d2a8d86 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""
Exercise the MDS's auto repair functions
"""

import logging
import time

from teuthology.exceptions import CommandFailedError
from tasks.cephfs.cephfs_test_case import CephFSTestCase


log = logging.getLogger(__name__)


# Arbitrary timeouts for operations involving restarting
# an MDS or waiting for it to come up
MDS_RESTART_GRACE = 60


class TestMDSAutoRepair(CephFSTestCase):
    def test_backtrace_repair(self):
        """
        MDS should verify/fix backtrace on fetch dirfrag
        """

        self.mount_a.run_shell(["mkdir", "testdir1"])
        self.mount_a.run_shell(["touch", "testdir1/testfile"])
        dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1"))

        # drop inodes caps
        self.mount_a.umount_wait()

        # flush journal entries to dirfrag objects, and expire journal
        self.fs.mds_asok(['flush', 'journal'])

        # Restart the MDS to drop the metadata cache (because we expired the journal,
        # nothing gets replayed into cache on restart)
        self.fs.rank_fail()
        self.fs.wait_for_daemons()

        # remove testdir1's backtrace
        self.fs.radosm(["rmxattr", dir_objname, "parent"])

        # readdir (fetch dirfrag) should fix testdir1's backtrace
        self.mount_a.mount_wait()
        self.mount_a.run_shell(["ls", "testdir1"])

        # flush journal entries to dirfrag objects
        self.fs.mds_asok(['flush', 'journal'])

        # check if backtrace exists
        self.fs.radosm(["getxattr", dir_objname, "parent"])

    def test_mds_readonly(self):
        """
        test if MDS behave correct when it's readonly
        """
        # operation should successd when MDS is not readonly
        self.mount_a.run_shell(["touch", "test_file1"])
        writer = self.mount_a.write_background(loop=True)

        time.sleep(10)
        self.assertFalse(writer.finished)

        # force MDS to read-only mode
        self.fs.mds_asok(['force_readonly'])
        time.sleep(10)

        # touching test file should fail
        try:
            self.mount_a.run_shell(["touch", "test_file1"])
        except CommandFailedError:
            pass
        else:
            self.assertTrue(False)

        # background writer also should fail
        self.assertTrue(writer.finished)

        # The MDS should report its readonly health state to the mon
        self.wait_for_health("MDS_READ_ONLY", timeout=30)

        # restart mds to make it writable
        self.fs.mds_fail_restart()
        self.fs.wait_for_daemons()

        self.wait_for_health_clear(timeout=30)