summaryrefslogtreecommitdiffstats
path: root/src/pybind/mgr/snap_schedule
diff options
context:
space:
mode:
Diffstat (limited to 'src/pybind/mgr/snap_schedule')
-rw-r--r--src/pybind/mgr/snap_schedule/fs/schedule.py39
-rw-r--r--src/pybind/mgr/snap_schedule/fs/schedule_client.py53
-rw-r--r--src/pybind/mgr/snap_schedule/module.py154
-rw-r--r--src/pybind/mgr/snap_schedule/tests/conftest.py37
4 files changed, 240 insertions, 43 deletions
diff --git a/src/pybind/mgr/snap_schedule/fs/schedule.py b/src/pybind/mgr/snap_schedule/fs/schedule.py
index 95e43b7e0..ba4390939 100644
--- a/src/pybind/mgr/snap_schedule/fs/schedule.py
+++ b/src/pybind/mgr/snap_schedule/fs/schedule.py
@@ -65,7 +65,7 @@ def parse_retention(retention: str) -> Dict[str, int]:
return ret
-RETENTION_MULTIPLIERS = ['n', 'M', 'h', 'd', 'w', 'm', 'y']
+RETENTION_MULTIPLIERS = ['n', 'm', 'h', 'd', 'w', 'M', 'Y']
TableRowT = Dict[str, Union[int, str]]
@@ -89,6 +89,7 @@ class Schedule(object):
rel_path: str,
start: Optional[str] = None,
subvol: Optional[str] = None,
+ group: Optional[str] = None,
retention_policy: str = '{}',
created: Optional[str] = None,
first: Optional[str] = None,
@@ -100,9 +101,15 @@ class Schedule(object):
) -> None:
self.fs = fs_name
self.subvol = subvol
+ self.group = group
self.path = path
self.rel_path = rel_path
self.schedule = schedule
+ # test to see if period and spec are valid
+ # this test will throw a ValueError exception if
+ # period is negative or zero
+ # spec is empty or other than n,m,h,d,w,M,Y
+ rep = self.repeat
self.retention = json.loads(retention_policy)
if start is None:
now = datetime.now(timezone.utc)
@@ -140,6 +147,7 @@ class Schedule(object):
cast(str, table_row['rel_path']),
cast(str, table_row['start']),
cast(str, table_row['subvol']),
+ cast(str, table_row['group_name']),
cast(str, table_row['retention']),
cast(str, table_row['created']),
cast(str, table_row['first']),
@@ -195,7 +203,7 @@ class Schedule(object):
ORDER BY until;'''
PROTO_GET_SCHEDULES = '''SELECT
- s.path, s.subvol, s.rel_path, sm.active,
+ s.path, s.subvol, s.group_name, s.rel_path, sm.active,
sm.schedule, s.retention, sm.start, sm.first, sm.last,
sm.last_pruned, sm.created, sm.created_count, sm.pruned_count
FROM schedules s
@@ -250,8 +258,8 @@ class Schedule(object):
return [cls._from_db_row(row, fs) for row in c.fetchall()]
INSERT_SCHEDULE = '''INSERT INTO
- schedules(path, subvol, retention, rel_path)
- Values(?, ?, ?, ?);'''
+ schedules(path, subvol, group_name, retention, rel_path)
+ Values(?, ?, ?, ?, ?);'''
INSERT_SCHEDULE_META = '''INSERT INTO
schedules_meta(schedule_id, start, created, repeat, schedule,
active)
@@ -265,6 +273,7 @@ class Schedule(object):
c = db.execute(self.INSERT_SCHEDULE,
(self.path,
self.subvol,
+ self.group,
json.dumps(self.retention),
self.rel_path,))
sched_id = c.lastrowid
@@ -396,8 +405,19 @@ class Schedule(object):
@property
def repeat(self) -> int:
- period, mult = self.parse_schedule(self.schedule)
- if mult == 'M':
+ period = -1
+ mult = ""
+ try:
+ period, mult = self.parse_schedule(self.schedule)
+ except ValueError:
+ raise ValueError('invalid schedule specified - period should be '
+ 'non-zero positive value and multiplier should '
+ 'be one of h,d,w,M,Y e.g. 1h or 4d etc.')
+ if period <= 0:
+ raise ValueError('invalid schedule specified - period must be a '
+ 'non-zero positive value e.g. 1h or 4d etc.')
+ # 'm' is only for developer testing of minute level snapshots
+ if mult == 'm':
return period * 60
elif mult == 'h':
return period * 60 * 60
@@ -405,8 +425,13 @@ class Schedule(object):
return period * 60 * 60 * 24
elif mult == 'w':
return period * 60 * 60 * 24 * 7
+ elif mult == 'M':
+ return period * 60 * 60 * 24 * 30
+ elif mult == 'Y':
+ return period * 60 * 60 * 24 * 365
else:
- raise ValueError(f'schedule multiplier "{mult}" not recognized')
+ raise ValueError('invalid schedule specified - multiplier should '
+ 'be one of h,d,w,M,Y')
UPDATE_LAST = '''UPDATE schedules_meta
SET
diff --git a/src/pybind/mgr/snap_schedule/fs/schedule_client.py b/src/pybind/mgr/snap_schedule/fs/schedule_client.py
index 28d54639a..8b199b632 100644
--- a/src/pybind/mgr/snap_schedule/fs/schedule_client.py
+++ b/src/pybind/mgr/snap_schedule/fs/schedule_client.py
@@ -79,12 +79,12 @@ def get_prune_set(candidates: Set[Tuple[cephfs.DirEntry, datetime]],
# NOTE: prune set has tz suffix stripped out.
("n", SNAPSHOT_TS_FORMAT),
# TODO remove M for release
- ("M", '%Y-%m-%d-%H_%M'),
+ ("m", '%Y-%m-%d-%H_%M'),
("h", '%Y-%m-%d-%H'),
("d", '%Y-%m-%d'),
("w", '%G-%V'),
- ("m", '%Y-%m'),
- ("y", '%Y'),
+ ("M", '%Y-%m'),
+ ("Y", '%Y'),
])
keep = []
if not retention:
@@ -180,6 +180,41 @@ class SnapSchedClient(CephfsClient):
def dump_on_update(self) -> None:
return self.mgr.get_module_option('dump_on_update')
+ def _create_snap_schedule_kv_db(self, db: sqlite3.Connection) -> None:
+ SQL = """
+ CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV (
+ key TEXT PRIMARY KEY,
+ value NOT NULL
+ ) WITHOUT ROWID;
+ INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1);
+ """
+ db.executescript(SQL)
+
+ def _get_snap_schedule_db_version(self, db: sqlite3.Connection) -> int:
+ SQL = """
+ SELECT value
+ FROM SnapScheduleModuleKV
+ WHERE key = '__snap_schedule_db_version';
+ """
+ cur = db.execute(SQL)
+ row = cur.fetchone()
+ assert row is not None
+ return int(row[0])
+
+ # add all upgrades here
+ def _upgrade_snap_schedule_db_schema(self, db: sqlite3.Connection) -> None:
+ # add a column to hold the subvolume group name
+ if self._get_snap_schedule_db_version(db) < 2:
+ SQL = """
+ ALTER TABLE schedules
+ ADD COLUMN group_name TEXT;
+ """
+ db.executescript(SQL)
+
+ # bump up the snap-schedule db version to 2
+ SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';"
+ db.execute(SQL, (2,))
+
def get_schedule_db(self, fs: str) -> DBConnectionManager:
dbinfo = None
self.conn_lock.acquire()
@@ -206,13 +241,15 @@ class SnapSchedClient(CephfsClient):
except rados.ObjectNotFound:
log.debug(f'No legacy schedule DB found in {fs}')
db.executescript(Schedule.CREATE_TABLES)
+ self._create_snap_schedule_kv_db(db)
+ self._upgrade_snap_schedule_db_schema(db)
self.sqlite_connections[fs] = DBInfo(fs, db)
dbinfo = self.sqlite_connections[fs]
self.conn_lock.release()
return DBConnectionManager(dbinfo)
def _is_allowed_repeat(self, exec_row: Dict[str, str], path: str) -> bool:
- if Schedule.parse_schedule(exec_row['schedule'])[1] == 'M':
+ if Schedule.parse_schedule(exec_row['schedule'])[1] == 'm':
if self.allow_minute_snaps:
log.debug(('Minute repeats allowed, '
f'scheduling snapshot on path {path}'))
@@ -370,12 +407,14 @@ class SnapSchedClient(CephfsClient):
def store_snap_schedule(self,
fs: str, path_: str,
args: Tuple[str, str, str, str,
- Optional[str], Optional[str]]) -> None:
+ Optional[str], Optional[str],
+ Optional[str]]) -> None:
sched = Schedule(*args)
log.debug(f'repeat is {sched.repeat}')
- if sched.parse_schedule(sched.schedule)[1] == 'M' and not self.allow_minute_snaps:
+ if sched.parse_schedule(sched.schedule)[1] == 'm' and not self.allow_minute_snaps:
log.error('not allowed')
- raise ValueError('no minute snaps allowed')
+ raise ValueError('invalid schedule specified - multiplier should '
+ 'be one of h,d,w,M,Y')
log.debug(f'attempting to add schedule {sched}')
with self.get_schedule_db(fs) as conn_mgr:
db = conn_mgr.dbinfo.db
diff --git a/src/pybind/mgr/snap_schedule/module.py b/src/pybind/mgr/snap_schedule/module.py
index b691572b6..d8f04a62b 100644
--- a/src/pybind/mgr/snap_schedule/module.py
+++ b/src/pybind/mgr/snap_schedule/module.py
@@ -6,7 +6,7 @@ LGPL2.1. See file COPYING.
import errno
import json
import sqlite3
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple, Union
from .fs.schedule_client import SnapSchedClient
from mgr_module import MgrModule, CLIReadCommand, CLIWriteCommand, Option
from mgr_util import CephfsConnectionException
@@ -37,6 +37,44 @@ class Module(MgrModule):
self._initialized = Event()
self.client = SnapSchedClient(self)
+ def _subvolume_exist(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> bool:
+ rc, subvolumes, err = self.remote('volumes', 'subvolume_ls', fs, group)
+ if rc == 0:
+ for svj in json.loads(subvolumes):
+ if subvol == svj['name']:
+ return True
+
+ return False
+
+ def _subvolume_flavor(self, fs: str, subvol: Union[str, None], group: Union[str, None]) -> str:
+ rc, subvol_info, err = self.remote('volumes', 'subvolume_info', fs, subvol, group)
+ svi_json = json.loads(subvol_info)
+ return svi_json.get('flavor', 'bad_flavor') # 1 or 2 etc.
+
+ def _resolve_subvolume_path(self, fs: str, path: str, subvol: Union[str, None], group: Union[str, None]) -> Tuple[int, str, str]:
+ if subvol is None and group is None:
+ return 0, path, ''
+
+ rc = -1
+ subvol_path = ''
+ if self._subvolume_exist(fs, subvol, group):
+ rc, subvol_path, err = self.remote('volumes', 'subvolume_getpath', fs, subvol, group)
+ if rc != 0:
+ return rc, '', f'Could not resolve subvol:{subvol} path in fs:{fs}'
+ else:
+ subvol_flavor = self._subvolume_flavor(fs, subvol, group)
+ if subvol_flavor == 1: # v1
+ return 0, subvol_path, f'Ignoring user specified path:{path} for subvol'
+ if subvol_flavor == 2: # v2
+ err = '';
+ if path != "/..":
+ err = f'Ignoring user specified path:{path} for subvol'
+ return 0, subvol_path + "/..", err
+
+ return -errno.EINVAL, '', f'Unhandled subvol flavor:{subvol_flavor}'
+ else:
+ return -errno.EINVAL, '', f'No such subvol: {group}::{subvol}'
+
@property
def _default_fs(self) -> Tuple[int, str, str]:
fs_map = self.get('fs_map')
@@ -53,11 +91,11 @@ class Module(MgrModule):
rc, fs, err = self._default_fs
if rc < 0:
return rc, fs, err
- if not self.has_fs(fs):
+ if not self._has_fs(fs):
return -errno.EINVAL, '', f"no such file system: {fs}"
return 0, fs, 'Success'
- def has_fs(self, fs_name: str) -> bool:
+ def _has_fs(self, fs_name: str) -> bool:
return fs_name in self.client.get_all_filesystems()
def serve(self) -> None:
@@ -71,6 +109,8 @@ class Module(MgrModule):
def snap_schedule_get(self,
path: str = '/',
fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None,
format: Optional[str] = 'plain') -> Tuple[int, str, str]:
'''
List current snapshot schedules
@@ -78,19 +118,28 @@ class Module(MgrModule):
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ errstr = 'Success'
try:
- ret_scheds = self.client.get_snap_schedules(fs, path)
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
+ ret_scheds = self.client.get_snap_schedules(fs, abs_path)
except CephfsConnectionException as e:
return e.to_tuple()
+ except Exception as e:
+ return -errno.EIO, '', str(e)
if format == 'json':
json_report = ','.join([ret_sched.report_json() for ret_sched in ret_scheds])
return 0, f'[{json_report}]', ''
+ self.log.info(errstr)
return 0, '\n===\n'.join([ret_sched.report() for ret_sched in ret_scheds]), ''
@CLIReadCommand('fs snap-schedule list')
def snap_schedule_list(self, path: str,
recursive: bool = False,
fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None,
format: Optional[str] = 'plain') -> Tuple[int, str, str]:
'''
Get current snapshot schedule for <path>
@@ -98,11 +147,17 @@ class Module(MgrModule):
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- scheds = self.client.list_snap_schedules(fs, path, recursive)
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
+ scheds = self.client.list_snap_schedules(fs, abs_path, recursive)
self.log.debug(f'recursive is {recursive}')
except CephfsConnectionException as e:
return e.to_tuple()
+ except Exception as e:
+ return -errno.EIO, '', str(e)
if not scheds:
if format == 'json':
output: Dict[str, str] = {}
@@ -112,7 +167,7 @@ class Module(MgrModule):
# json_list = ','.join([sched.json_list() for sched in scheds])
schedule_list = [sched.schedule for sched in scheds]
retention_list = [sched.retention for sched in scheds]
- out = {'path': path, 'schedule': schedule_list, 'retention': retention_list}
+ out = {'path': abs_path, 'schedule': schedule_list, 'retention': retention_list}
return 0, json.dumps(out), ''
return 0, '\n'.join([str(sched) for sched in scheds]), ''
@@ -121,23 +176,27 @@ class Module(MgrModule):
path: str,
snap_schedule: str,
start: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Set a snapshot schedule for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
- subvol = None
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.store_snap_schedule(fs,
abs_path,
(abs_path, snap_schedule,
- fs, path, start, subvol))
- suc_msg = f'Schedule set for path {path}'
+ fs, abs_path, start, subvol, group))
+ suc_msg = f'Schedule set for path {abs_path}'
except sqlite3.IntegrityError:
- existing_scheds = self.client.get_snap_schedules(fs, path)
+ existing_scheds = self.client.get_snap_schedules(fs, abs_path)
report = [s.report() for s in existing_scheds]
error_msg = f'Found existing schedule {report}'
self.log.error(error_msg)
@@ -146,6 +205,8 @@ class Module(MgrModule):
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
+ except Exception as e:
+ return -errno.EIO, '', str(e)
return 0, suc_msg, ''
@CLIWriteCommand('fs snap-schedule remove')
@@ -153,36 +214,48 @@ class Module(MgrModule):
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Remove a snapshot schedule for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.rm_snap_schedule(fs, abs_path, repeat, start)
except ValueError as e:
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
- return 0, 'Schedule removed for path {}'.format(path), ''
+ except Exception as e:
+ return -errno.EIO, '', str(e)
+ return 0, 'Schedule removed for path {}'.format(abs_path), ''
@CLIWriteCommand('fs snap-schedule retention add')
def snap_schedule_retention_add(self,
path: str,
retention_spec_or_period: str,
retention_count: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Set a retention specification for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.add_retention_spec(fs, abs_path,
retention_spec_or_period,
retention_count)
@@ -190,69 +263,92 @@ class Module(MgrModule):
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
- return 0, 'Retention added to path {}'.format(path), ''
+ except Exception as e:
+ return -errno.EIO, '', str(e)
+ return 0, 'Retention added to path {}'.format(abs_path), ''
@CLIWriteCommand('fs snap-schedule retention remove')
def snap_schedule_retention_rm(self,
path: str,
retention_spec_or_period: str,
retention_count: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Remove a retention specification for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.rm_retention_spec(fs, abs_path,
retention_spec_or_period,
retention_count)
- except CephfsConnectionException as e:
- return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
- return 0, 'Retention removed from path {}'.format(path), ''
+ except CephfsConnectionException as e:
+ return e.to_tuple()
+ except Exception as e:
+ return -errno.EIO, '', str(e)
+ return 0, 'Retention removed from path {}'.format(abs_path), ''
@CLIWriteCommand('fs snap-schedule activate')
def snap_schedule_activate(self,
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Activate a snapshot schedule for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.activate_snap_schedule(fs, abs_path, repeat, start)
except ValueError as e:
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
- return 0, 'Schedule activated for path {}'.format(path), ''
+ except Exception as e:
+ return -errno.EIO, '', str(e)
+ return 0, 'Schedule activated for path {}'.format(abs_path), ''
@CLIWriteCommand('fs snap-schedule deactivate')
def snap_schedule_deactivate(self,
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
- fs: Optional[str] = None) -> Tuple[int, str, str]:
+ fs: Optional[str] = None,
+ subvol: Optional[str] = None,
+ group: Optional[str] = None) -> Tuple[int, str, str]:
'''
Deactivate a snapshot schedule for <path>
'''
rc, fs, err = self._validate_fs(fs)
if rc < 0:
return rc, fs, err
+ abs_path = ""
try:
- abs_path = path
+ rc, abs_path, errstr = self._resolve_subvolume_path(fs, path, subvol, group)
+ if rc != 0:
+ return rc, '', errstr
self.client.deactivate_snap_schedule(fs, abs_path, repeat, start)
except ValueError as e:
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
- return 0, 'Schedule deactivated for path {}'.format(path), ''
+ except Exception as e:
+ return -errno.EIO, '', str(e)
+ return 0, 'Schedule deactivated for path {}'.format(abs_path), ''
diff --git a/src/pybind/mgr/snap_schedule/tests/conftest.py b/src/pybind/mgr/snap_schedule/tests/conftest.py
index 35255b8d4..92d4c9bc1 100644
--- a/src/pybind/mgr/snap_schedule/tests/conftest.py
+++ b/src/pybind/mgr/snap_schedule/tests/conftest.py
@@ -31,4 +31,41 @@ def db():
db.row_factory = sqlite3.Row
db.execute("PRAGMA FOREIGN_KEYS = 1")
db.executescript(Schedule.CREATE_TABLES)
+ _create_snap_schedule_kv_db(db)
+ _upgrade_snap_schedule_db_schema(db)
return db
+
+def _create_snap_schedule_kv_db(db):
+ SQL = """
+ CREATE TABLE IF NOT EXISTS SnapScheduleModuleKV (
+ key TEXT PRIMARY KEY,
+ value NOT NULL
+ ) WITHOUT ROWID;
+ INSERT OR IGNORE INTO SnapScheduleModuleKV (key, value) VALUES ('__snap_schedule_db_version', 1);
+ """
+ db.executescript(SQL)
+
+def _get_snap_schedule_db_version(db):
+ SQL = """
+ SELECT value
+ FROM SnapScheduleModuleKV
+ WHERE key = '__snap_schedule_db_version';
+ """
+ cur = db.execute(SQL)
+ row = cur.fetchone()
+ assert row is not None
+ return int(row[0])
+
+# add all upgrades here
+def _upgrade_snap_schedule_db_schema(db):
+ # add a column to hold the subvolume group name
+ if _get_snap_schedule_db_version(db) < 2:
+ SQL = """
+ ALTER TABLE schedules
+ ADD COLUMN group_name TEXT;
+ """
+ db.executescript(SQL)
+
+ # bump up the snap-schedule db version to 2
+ SQL = "UPDATE OR ROLLBACK SnapScheduleModuleKV SET value = ? WHERE key = '__snap_schedule_db_version';"
+ db.execute(SQL, (2,))