diff options
Diffstat (limited to 'src/pybind/mgr/snap_schedule')
| -rw-r--r-- | src/pybind/mgr/snap_schedule/fs/schedule.py | 39 | ||||
| -rw-r--r-- | src/pybind/mgr/snap_schedule/fs/schedule_client.py | 53 | ||||
| -rw-r--r-- | src/pybind/mgr/snap_schedule/module.py | 154 | ||||
| -rw-r--r-- | src/pybind/mgr/snap_schedule/tests/conftest.py | 37 |
4 files changed, 240 insertions, 43 deletions
diff --git a/src/pybind/mgr/snap_schedule/fs/schedule.py b/src/pybind/mgr/snap_schedule/fs/schedule.py index 95e43b7e0..ba4390939 100644 --- a/src/pybind/mgr/snap_schedule/fs/schedule.py +++ b/src/pybind/mgr/snap_schedule/fs/schedule.py @@ -65,7 +65,7 @@ def parse_retention(retention: str) -> Dict[str, int]: return ret -RETENTION_MULTIPLIERS = ['n', 'M', 'h', 'd', 'w', 'm', 'y'] +RETENTION_MULTIPLIERS = ['n', 'm', 'h', 'd', 'w', 'M', 'Y'] TableRowT = Dict[str, Union[int, str]] @@ -89,6 +89,7 @@ class Schedule(object): rel_path: str, start: Optional[str] = None, subvol: Optional[str] = None, + group: Optional[str] = None, retention_policy: str = '{}', created: Optional[str] = None, first: Optional[str] = None, @@ -100,9 +101,15 @@ class Schedule(object): ) -> None: self.fs = fs_name self.subvol = subvol + self.group = group self.path = path self.rel_path = rel_path self.schedule = schedule + # test to see if period and spec are valid + # this test will throw a ValueError exception if + # period is negative or zero + # spec is empty or other than n,m,h,d,w,M,Y + rep = self.repeat self.retention = json.loads(retention_policy) if start is None: now = datetime.now(timezone.utc) @@ -140,6 +147,7 @@ class Schedule(object): cast(str, table_row['rel_path']), cast(str, table_row['start']), cast(str, table_row['subvol']), + cast(str, table_row['group_name']), cast(str, table_row['retention']), cast(str, table_row['created']), cast(str, table_row['first']), @@ -195,7 +203,7 @@ class Schedule(object): ORDER BY until;''' PROTO_GET_SCHEDULES = '''SELECT - s.path, s.subvol, s.rel_path, sm.active, + s.path, s.subvol, s.group_name, s.rel_path, sm.active, sm.schedule, s.retention, sm.start, sm.first, sm.last, sm.last_pruned, sm.created, sm.created_count, sm.pruned_count FROM schedules s @@ -250,8 +258,8 @@ class Schedule(object): return [cls._from_db_row(row, fs) for row in c.fetchall()] INSERT_SCHEDULE = '''INSERT INTO - schedules(path, subvol, retention, rel_path) - Values(?, ?, ?, ?);''' + schedules(path, subvol, group_name, retention, rel_path) + Values(?, ?, ?, ?, ?);''' INSERT_SCHEDULE_META = '''INSERT INTO schedules_meta(schedule_id, start, created, repeat, schedule, active) @@ -265,6 +273,7 @@ class Schedule(object): c = db.execute(self.INSERT_SCHEDULE, (self.path, self.subvol, + self.group, json.dumps(self.retention), self.rel_path,)) sched_id = c.lastrowid @@ -396,8 +405,19 @@ class Schedule(object): @property def repeat(self) -> int: - period, mult = self.parse_schedule(self.schedule) - if mult == 'M': + period = -1 + mult = "" + try: + period, mult = self.parse_schedule(self.schedule) + except ValueError: + raise ValueError('invalid schedule specified - period should be ' + 'non-zero positive value and multiplier should ' + 'be one of h,d,w,M,Y e.g. 1h or 4d etc.') + if period <= 0: + raise ValueError('invalid schedule specified - period must be a ' + 'non-zero positive value e.g. 1h or 4d etc.') + # 'm' is only for developer testing of minute level snapshots + if mult == 'm': return period * 60 elif mult == 'h': return period * 60 * 60 @@ -405,8 +425,13 @@ class Schedule(object): return period * 60 * 60 * 24 elif mult == 'w': return period * 60 * 60 * 24 * 7 + elif mult == 'M': + return period * 60 * 60 * 24 * 30 + elif mult == 'Y': + return period * 60 * 60 * 24 * 365 else: - raise ValueError(f'schedule multiplier "{mult}" not recognized') + raise ValueError('invalid schedule specified - multiplier should ' + 'be one of h,d,w,M,Y') UPDATE_LAST = '''UPDATE schedules_meta SET diff --git a/src/pybind/mgr/snap_schedule/fs/schedule_client.py b/src/pybind/mgr/snap_schedule/fs/schedule_client.py index 28d54639a..8b199b632 100644 --- a/src/pybind/mgr/snap_schedule/fs/schedule_client.py +++ b/src/pybind/mgr/snap_schedule/fs/schedule_client.py @@ -79,12 +79,12 @@ def get_prune_set(candidates: Set[Tuple[cephfs.DirEntry, datetime]], # NOTE: prune set has tz suffix stripped out. ("n", SNAPSHOT_TS_FORMAT), # TODO remove M for release - ("M", '%Y-%m-%d-%H_%M'), + ("m", '%Y-%m-%d-%H_%M'), ("h", '%Y-%m-%d-%H'), ("d", '%Y-%m-%d'), ("w", '%G-%V'), - ("m", '%Y-%m'), - ("y", '%Y'), + ("M", '%Y-%m'), + ("Y", '%Y'), ]) keep = [] if not retention: @@ -180,6 +180,41 @@ class SnapSchedClient(CephfsClient): def dump_on_update(self) -> None: return self.mgr.get_module_option('dump_on_update') + def _create_snap_schedule_kv_db(self, db: sqlite3.Connection) -> None: + SQL = """ + CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV ( + key TEXT PRIMARY KEY, + value NOT NULL + ) WITHOUT ROWID; + INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1); + """ + db.executescript(SQL) + + def _get_snap_schedule_db_version(self, db: sqlite3.Connection) -> int: + SQL = """ + SELECT value + FROM SnapScheduleModuleKV + WHERE key = '__snap_schedule_db_version'; + """ + cur = db.execute(SQL) + row = cur.fetchone() + assert row is not None + return int(row[0]) + + # add all upgrades here + def _upgrade_snap_schedule_db_schema(self, db: sqlite3.Connection) -> None: + # add a column to hold the subvolume group name + if self._get_snap_schedule_db_version(db) < 2: + SQL = """ + ALTER TABLE schedules + ADD COLUMN group_name TEXT; + """ + db.executescript(SQL) + + # bump up the snap-schedule db version to 2 + SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';" + db.execute(SQL, (2,)) + def get_schedule_db(self, fs: str) -> DBConnectionManager: dbinfo = None self.conn_lock.acquire() @@ -206,13 +241,15 @@ class SnapSchedClient(CephfsClient): except rados.ObjectNotFound: log.debug(f'No legacy schedule DB found in {fs}') db.executescript(Schedule.CREATE_TABLES) + self._create_snap_schedule_kv_db(db) + self._upgrade_snap_schedule_db_schema(db) self.sqlite_connections[fs] = DBInfo(fs, db) dbinfo = self.sqlite_connections[fs] self.conn_lock.release() return DBConnectionManager(dbinfo) def _is_allowed_repeat(self, exec_row: Dict[str, str], path: str) -> bool: - if Schedule.parse_schedule(exec_row['schedule'])[1] == 'M': + if Schedule.parse_schedule(exec_row['schedule'])[1] == 'm': if self.allow_minute_snaps: log.debug(('Minute repeats allowed, ' f'scheduling snapshot on path {path}')) @@ -370,12 +407,14 @@ class SnapSchedClient(CephfsClient): def store_snap_schedule(self, fs: str, path_: str, args: Tuple[str, str, str, str, - Optional[str], Optional[str]]) -> None: + Optional[str], Optional[str], + Optional[str]]) -> None: sched = Schedule(*args) log.debug(f'repeat is {sched.repeat}') - if sched.parse_schedule(sched.schedule)[1] == 'M' and not self.allow_minute_snaps: + if sched.parse_schedule(sched.schedule)[1] == 'm' and not self.allow_minute_snaps: log.error('not allowed') - raise ValueError('no minute snaps allowed') + raise ValueError('invalid schedule specified - multiplier should ' + 'be one of h,d,w,M,Y') log.debug(f'attempting to add schedule {sched}') with self.get_schedule_db(fs) as conn_mgr: db = conn_mgr.dbinfo.db diff --git a/src/pybind/mgr/snap_schedule/module.py b/src/pybind/mgr/snap_schedule/module.py index b691572b6..d8f04a62b 100644 --- a/src/pybind/mgr/snap_schedule/module.py +++ b/src/pybind/mgr/snap_schedule/module.py @@ -6,7 +6,7 @@ LGPL2.1. See file COPYING. import errno import json import sqlite3 -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Union from .fs.schedule_client import SnapSchedClient from mgr_module import MgrModule, CLIReadCommand, CLIWriteCommand, Option from mgr_util import CephfsConnectionException @@ -37,6 +37,44 @@ class Module(MgrModule): self._initialized = Event() self.client = SnapSchedClient(self) + def _subvolume_exist(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> bool: + rc, subvolumes, err = self.remote('volumes', 'subvolume_ls', fs, group) + if rc == 0: + for svj in json.loads(subvolumes): + if subvol == svj['name']: + return True + + return False + + def _subvolume_flavor(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> str: + rc, subvol_info, err = self.remote('volumes', 'subvolume_info', fs, subvol, group) + svi_json = json.loads(subvol_info) + return svi_json.get('flavor', 'bad_flavor') # 1 or 2 etc. + + def _resolve_subvolume_path(self, fs: str, path: str, subvol: Union[str, None], group: Union[str, None]) -> Tuple[int, str, str]: + if subvol is None and group is None: + return 0, path, '' + + rc = -1 + subvol_path = '' + if self._subvolume_exist(fs, subvol, group): + rc, subvol_path, err = self.remote('volumes', 'subvolume_getpath', fs, subvol, group) + if rc != 0: + return rc, '', f'Could not resolve subvol:{subvol} path in fs:{fs}' + else: + subvol_flavor = self._subvolume_flavor(fs, subvol, group) + if subvol_flavor == 1: # v1 + return 0, subvol_path, f'Ignoring user specified path:{path} for subvol' + if subvol_flavor == 2: # v2 + err = ''; + if path != "/..": + err = f'Ignoring user specified path:{path} for subvol' + return 0, subvol_path + "/..", err + + return -errno.EINVAL, '', f'Unhandled subvol flavor:{subvol_flavor}' + else: + return -errno.EINVAL, '', f'No such subvol: {group}::{subvol}' + @property def _default_fs(self) -> Tuple[int, str, str]: fs_map = self.get('fs_map') @@ -53,11 +91,11 @@ class Module(MgrModule): rc, fs, err = self._default_fs if rc < 0: return rc, fs, err - if not self.has_fs(fs): + if not self._has_fs(fs): return -errno.EINVAL, '', f"no such file system: {fs}" return 0, fs, 'Success' - def has_fs(self, fs_name: str) -> bool: + def _has_fs(self, fs_name: str) -> bool: return fs_name in self.client.get_all_filesystems() def serve(self) -> None: @@ -71,6 +109,8 @@ class Module(MgrModule): def snap_schedule_get(self, path: str = '/', fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None, format: Optional[str] = 'plain') -> Tuple[int, str, str]: ''' List current snapshot schedules @@ -78,19 +118,28 @@ class Module(MgrModule): rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + errstr = 'Success' try: - ret_scheds = self.client.get_snap_schedules(fs, path) + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr + ret_scheds = self.client.get_snap_schedules(fs, abs_path) except CephfsConnectionException as e: return e.to_tuple() + except Exception as e: + return -errno.EIO, '', str(e) if format == 'json': json_report = ','.join([ret_sched.report_json() for ret_sched in ret_scheds]) return 0, f'[{json_report}]', '' + self.log.info(errstr) return 0, '\n===\n'.join([ret_sched.report() for ret_sched in ret_scheds]), '' @CLIReadCommand('fs snap-schedule list') def snap_schedule_list(self, path: str, recursive: bool = False, fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None, format: Optional[str] = 'plain') -> Tuple[int, str, str]: ''' Get current snapshot schedule for <path> @@ -98,11 +147,17 @@ class Module(MgrModule): rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - scheds = self.client.list_snap_schedules(fs, path, recursive) + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr + scheds = self.client.list_snap_schedules(fs, abs_path, recursive) self.log.debug(f'recursive is {recursive}') except CephfsConnectionException as e: return e.to_tuple() + except Exception as e: + return -errno.EIO, '', str(e) if not scheds: if format == 'json': output: Dict[str, str] = {} @@ -112,7 +167,7 @@ class Module(MgrModule): # json_list = ','.join([sched.json_list() for sched in scheds]) schedule_list = [sched.schedule for sched in scheds] retention_list = [sched.retention for sched in scheds] - out = {'path': path, 'schedule': schedule_list, 'retention': retention_list} + out = {'path': abs_path, 'schedule': schedule_list, 'retention': retention_list} return 0, json.dumps(out), '' return 0, '\n'.join([str(sched) for sched in scheds]), '' @@ -121,23 +176,27 @@ class Module(MgrModule): path: str, snap_schedule: str, start: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Set a snapshot schedule for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path - subvol = None + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.store_snap_schedule(fs, abs_path, (abs_path, snap_schedule, - fs, path, start, subvol)) - suc_msg = f'Schedule set for path {path}' + fs, abs_path, start, subvol, group)) + suc_msg = f'Schedule set for path {abs_path}' except sqlite3.IntegrityError: - existing_scheds = self.client.get_snap_schedules(fs, path) + existing_scheds = self.client.get_snap_schedules(fs, abs_path) report = [s.report() for s in existing_scheds] error_msg = f'Found existing schedule {report}' self.log.error(error_msg) @@ -146,6 +205,8 @@ class Module(MgrModule): return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() + except Exception as e: + return -errno.EIO, '', str(e) return 0, suc_msg, '' @CLIWriteCommand('fs snap-schedule remove') @@ -153,36 +214,48 @@ class Module(MgrModule): path: str, repeat: Optional[str] = None, start: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Remove a snapshot schedule for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.rm_snap_schedule(fs, abs_path, repeat, start) except ValueError as e: return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() - return 0, 'Schedule removed for path {}'.format(path), '' + except Exception as e: + return -errno.EIO, '', str(e) + return 0, 'Schedule removed for path {}'.format(abs_path), '' @CLIWriteCommand('fs snap-schedule retention add') def snap_schedule_retention_add(self, path: str, retention_spec_or_period: str, retention_count: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Set a retention specification for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.add_retention_spec(fs, abs_path, retention_spec_or_period, retention_count) @@ -190,69 +263,92 @@ class Module(MgrModule): return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() - return 0, 'Retention added to path {}'.format(path), '' + except Exception as e: + return -errno.EIO, '', str(e) + return 0, 'Retention added to path {}'.format(abs_path), '' @CLIWriteCommand('fs snap-schedule retention remove') def snap_schedule_retention_rm(self, path: str, retention_spec_or_period: str, retention_count: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Remove a retention specification for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.rm_retention_spec(fs, abs_path, retention_spec_or_period, retention_count) - except CephfsConnectionException as e: - return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) - return 0, 'Retention removed from path {}'.format(path), '' + except CephfsConnectionException as e: + return e.to_tuple() + except Exception as e: + return -errno.EIO, '', str(e) + return 0, 'Retention removed from path {}'.format(abs_path), '' @CLIWriteCommand('fs snap-schedule activate') def snap_schedule_activate(self, path: str, repeat: Optional[str] = None, start: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Activate a snapshot schedule for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.activate_snap_schedule(fs, abs_path, repeat, start) except ValueError as e: return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() - return 0, 'Schedule activated for path {}'.format(path), '' + except Exception as e: + return -errno.EIO, '', str(e) + return 0, 'Schedule activated for path {}'.format(abs_path), '' @CLIWriteCommand('fs snap-schedule deactivate') def snap_schedule_deactivate(self, path: str, repeat: Optional[str] = None, start: Optional[str] = None, - fs: Optional[str] = None) -> Tuple[int, str, str]: + fs: Optional[str] = None, + subvol: Optional[str] = None, + group: Optional[str] = None) -> Tuple[int, str, str]: ''' Deactivate a snapshot schedule for <path> ''' rc, fs, err = self._validate_fs(fs) if rc < 0: return rc, fs, err + abs_path = "" try: - abs_path = path + rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group) + if rc != 0: + return rc, '', errstr self.client.deactivate_snap_schedule(fs, abs_path, repeat, start) except ValueError as e: return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() - return 0, 'Schedule deactivated for path {}'.format(path), '' + except Exception as e: + return -errno.EIO, '', str(e) + return 0, 'Schedule deactivated for path {}'.format(abs_path), '' diff --git a/src/pybind/mgr/snap_schedule/tests/conftest.py b/src/pybind/mgr/snap_schedule/tests/conftest.py index 35255b8d4..92d4c9bc1 100644 --- a/src/pybind/mgr/snap_schedule/tests/conftest.py +++ b/src/pybind/mgr/snap_schedule/tests/conftest.py @@ -31,4 +31,41 @@ def db(): db.row_factory = sqlite3.Row db.execute("PRAGMA FOREIGN_KEYS = 1") db.executescript(Schedule.CREATE_TABLES) + _create_snap_schedule_kv_db(db) + _upgrade_snap_schedule_db_schema(db) return db + +def _create_snap_schedule_kv_db(db): + SQL = """ + CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV ( + key TEXT PRIMARY KEY, + value NOT NULL + ) WITHOUT ROWID; + INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1); + """ + db.executescript(SQL) + +def _get_snap_schedule_db_version(db): + SQL = """ + SELECT value + FROM SnapScheduleModuleKV + WHERE key = '__snap_schedule_db_version'; + """ + cur = db.execute(SQL) + row = cur.fetchone() + assert row is not None + return int(row[0]) + +# add all upgrades here +def _upgrade_snap_schedule_db_schema(db): + # add a column to hold the subvolume group name + if _get_snap_schedule_db_version(db) < 2: + SQL = """ + ALTER TABLE schedules + ADD COLUMN group_name TEXT; + """ + db.executescript(SQL) + + # bump up the snap-schedule db version to 2 + SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';" + db.execute(SQL, (2,)) |