diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-23 16:45:13 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-23 16:45:13 +0000 |
| commit | 389020e14594e4894e28d1eb9103c210b142509e (patch) | |
| tree | 2ba734cdd7a243f46dda7c3d0cc88c2293d9699f /src/ceph-volume | |
| parent | Adding upstream version 18.2.2. (diff) | |
| download | ceph-389020e14594e4894e28d1eb9103c210b142509e.tar.xz ceph-389020e14594e4894e28d1eb9103c210b142509e.zip | |
Adding upstream version 18.2.3.upstream/18.2.3upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/ceph-volume')
102 files changed, 277 insertions, 665 deletions
diff --git a/src/ceph-volume/ceph_volume/__init__.py b/src/ceph-volume/ceph_volume/__init__.py index dad83c95b..d0014284d 100644 --- a/src/ceph-volume/ceph_volume/__init__.py +++ b/src/ceph-volume/ceph_volume/__init__.py @@ -14,8 +14,9 @@ class UnloadedConfig(object): def __getattr__(self, *a): raise RuntimeError("No valid ceph configuration file was loaded.") -conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path']) +conf = namedtuple('config', ['ceph', 'cluster', 'verbosity', 'path', 'log_path', 'dmcrypt_no_workqueue']) conf.ceph = UnloadedConfig() +conf.dmcrypt_no_workqueue = None __version__ = "1.0.0" diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py index feb91053b..17c66194c 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -69,6 +69,8 @@ def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False): raise RuntimeError('could not find a bluestore OSD to activate') is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1' + if is_encrypted and conf.dmcrypt_no_workqueue is None: + encryption_utils.set_dmcrypt_no_workqueue() dmcrypt_secret = None osd_id = osd_block_lv.tags['ceph.osd_id'] conf.cluster = osd_block_lv.tags['ceph.cluster_name'] diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 69a3f672b..2118ce47a 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -256,7 +256,7 @@ class Batch(object): ) parser.add_argument( '--dmcrypt', - action='store_true', + action=arg_validators.DmcryptAction, help='Enable device encryption via dm-crypt', ) parser.add_argument( diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py index 35e53181a..198ba9417 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -73,7 +73,7 @@ common_args = { 'default': "", }, '--dmcrypt': { - 'action': 'store_true', + 'action': arg_validators.DmcryptAction, 'help': 'Enable device encryption via dm-crypt', }, '--no-systemd': { diff --git a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py index 64589a2d6..474b479de 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py @@ -167,7 +167,11 @@ class VolumeTagTracker(object): aux_dev.lv_api.set_tags(tags) def remove_lvs(self, source_devices, target_type): - remaining_devices = [self.data_device, self.db_device, self.wal_device] + remaining_devices = [self.data_device] + if self.db_device: + remaining_devices.append(self.db_device) + if self.wal_device: + remaining_devices.append(self.wal_device) outdated_tags = [] for device, type in source_devices: diff --git a/src/ceph-volume/ceph_volume/devices/lvm/zap.py b/src/ceph-volume/ceph_volume/devices/lvm/zap.py index d4d78ad01..f512a3bde 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/zap.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/zap.py @@ -10,6 +10,7 @@ from ceph_volume.api import lvm as api from ceph_volume.util import system, encryption, disk, arg_validators, str_to_int, merge_dict from ceph_volume.util.device import Device from ceph_volume.systemd import systemctl +from typing import List logger = logging.getLogger(__name__) mlogger = terminal.MultiLogger(__name__) @@ -201,8 +202,11 @@ class Zap(object): """ if device.is_encrypted: # find the holder - holders = [ - '/dev/%s' % holder for holder in device.sys_api.get('holders', []) + pname = device.sys_api.get('parent') + devname = device.sys_api.get('devname') + parent_device = Device(f'/dev/{pname}') + holders: List[str] = [ + f'/dev/{holder}' for holder in parent_device.sys_api['partitions'][devname]['holders'] ] for mapper_uuid in os.listdir('/dev/mapper'): mapper_path = os.path.join('/dev/mapper', mapper_uuid) diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py index 89ee285be..4863b9e18 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/common.py +++ b/src/ceph-volume/ceph_volume/devices/raw/common.py @@ -46,7 +46,7 @@ def create_parser(prog, description): ) parser.add_argument( '--dmcrypt', - action='store_true', + action=arg_validators.DmcryptAction, help='Enable device encryption via dm-crypt', ) parser.add_argument( diff --git a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py index 7e516f3d2..91e6155f3 100644 --- a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py +++ b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_migrate.py @@ -2466,6 +2466,93 @@ Example calls for supported scenarios: '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal'] @patch('os.getuid') + def test_migrate_wal_to_db(self, + m_getuid, + monkeypatch, + capsys): + m_getuid.return_value = 0 + + source_tags = 'ceph.osd_id=2,ceph.type=data,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + source_wal_tags = 'ceph.osd_id=2,ceph.type=wal,ceph.osd_fsid=1234,' \ + 'ceph.cluster_name=ceph,' \ + 'ceph.wal_uuid=waluuid,ceph.wal_device=wal_dev' + + data_vol = api.Volume(lv_name='volume1', + lv_uuid='datauuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv1', + lv_tags=source_tags) + + wal_vol = api.Volume(lv_name='volume3', + lv_uuid='waluuid', + vg_name='vg', + lv_path='/dev/VolGroup/lv3', + lv_tags=source_wal_tags) + + self.mock_single_volumes = { + '/dev/VolGroup/lv1': data_vol, + '/dev/VolGroup/lv3': wal_vol, + } + monkeypatch.setattr(migrate.api, 'get_single_lv', + self.mock_get_single_lv) + + self.mock_volume = data_vol + monkeypatch.setattr(api, 'get_lv_by_fullname', + self.mock_get_lv_by_fullname) + + self.mock_process_input = [] + monkeypatch.setattr(process, 'call', self.mock_process) + + devices = [] + devices.append([Device('/dev/VolGroup/lv1'), 'block']) + devices.append([Device('/dev/VolGroup/lv3'), 'wal']) + + monkeypatch.setattr(migrate, 'find_associated_devices', + lambda osd_id, osd_fsid: devices) + + monkeypatch.setattr("ceph_volume.systemd.systemctl.osd_is_active", + lambda id: False) + + monkeypatch.setattr(migrate, 'get_cluster_name', + lambda osd_id, osd_fsid: 'ceph') + monkeypatch.setattr(system, 'chown', lambda path: 0) + m = migrate.Migrate(argv=[ + '--osd-id', '2', + '--osd-fsid', '1234', + '--from', 'wal', + '--target', 'vgname/data']) + + m.main() + + n = len(self.mock_process_input) + assert n >= 1 + for s in self.mock_process_input: + print(s) + + assert self. mock_process_input[n-3] == [ + 'lvchange', + '--deltag', 'ceph.osd_id=2', + '--deltag', 'ceph.type=wal', + '--deltag', 'ceph.osd_fsid=1234', + '--deltag', 'ceph.cluster_name=ceph', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv3'] + assert self. mock_process_input[n-2] == [ + 'lvchange', + '--deltag', 'ceph.wal_uuid=waluuid', + '--deltag', 'ceph.wal_device=wal_dev', + '/dev/VolGroup/lv1'] + assert self. mock_process_input[n-1] == [ + 'ceph-bluestore-tool', + '--path', '/var/lib/ceph/osd/ceph-2', + '--dev-target', '/var/lib/ceph/osd/ceph-2/block', + '--command', 'bluefs-bdev-migrate', + '--devs-source', '/var/lib/ceph/osd/ceph-2/block.wal'] + + @patch('os.getuid') def test_migrate_data_wal_to_db_encrypted(self, m_getuid, monkeypatch, diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/group_vars/all index c3808c1d7..c3808c1d7 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/setup.yml index 8cf11d4ef..8cf11d4ef 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/test.yml index 66d44c728..66d44c728 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt-explicit/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt-explicit/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/group_vars/all index c3808c1d7..c3808c1d7 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/setup.yml index 8cf11d4ef..8cf11d4ef 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/test.yml index aa867bcde..aa867bcde 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-dmcrypt/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-dmcrypt/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/group_vars/all index c3808c1d7..c3808c1d7 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/setup.yml index 8cf11d4ef..8cf11d4ef 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/test.yml index 66d44c728..66d44c728 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type-explicit/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type-explicit/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/group_vars/all index c3808c1d7..c3808c1d7 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/setup.yml index 8cf11d4ef..8cf11d4ef 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/test.yml index aa867bcde..aa867bcde 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/mixed-type/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/mixed-type/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/group_vars/all index 1e6ea0080..1e6ea0080 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/setup.yml index 30874dfbb..30874dfbb 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/test.yml index aa867bcde..aa867bcde 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type-dmcrypt/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type-dmcrypt/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/group_vars/all index 1e6ea0080..1e6ea0080 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/hosts b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/setup.yml index 30874dfbb..30874dfbb 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/test.yml index aa867bcde..aa867bcde 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/test_zap.yml index cb969fa1d..cb969fa1d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/test_zap.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/centos8/bluestore/single-type/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/centos/bluestore/single-type/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test.yml index 5d5bc59f2..17f200c9d 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test.yml @@ -14,10 +14,10 @@ tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + command: "ceph osd down osd.{{ item }}" with_items: "{{ osd_ids }}" - name: purge osds - command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it" + command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it" with_items: "{{ osd_ids }}" - hosts: osds @@ -25,18 +25,18 @@ tasks: - name: zap devices used for OSDs - command: "ceph-volume --cluster {{ cluster }} lvm zap {{ item }} --destroy" + command: "ceph-volume lvm zap {{ item }} --destroy" with_items: "{{ devices }}" environment: CEPH_VOLUME_DEBUG: 1 - name: batch create devices again - command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" + command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" environment: CEPH_VOLUME_DEBUG: 1 - name: ensure batch create is idempotent - command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" + command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" register: batch_cmd failed_when: false environment: @@ -50,7 +50,7 @@ - "'strategy changed' not in batch_cmd.stderr" - name: run batch --report to see if devices get filtered - command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" + command: "ceph-volume lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices | join(' ') }}" register: report_cmd failed_when: false environment: diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml index 1ff0acc9d..2581f5c46 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_explicit.yml @@ -14,10 +14,10 @@ tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + command: "ceph osd down osd.{{ item }}" with_items: "{{ osd_ids }}" - name: purge osds - command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it" + command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it" with_items: "{{ osd_ids }}" - hosts: osds @@ -27,18 +27,18 @@ tasks: - name: zap devices used for OSDs - command: "ceph-volume --cluster {{ cluster }} lvm zap {{ item }} --destroy" + command: "ceph-volume lvm zap {{ item }} --destroy" with_items: "{{ devices }}" environment: CEPH_VOLUME_DEBUG: 1 - name: batch create devices again - command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" + command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" environment: CEPH_VOLUME_DEBUG: 1 - name: ensure batch create is idempotent when all data devices are filtered - command: "ceph-volume --cluster {{ cluster }} lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" + command: "ceph-volume lvm batch --yes --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" register: batch_cmd failed_when: false environment: @@ -51,7 +51,7 @@ - batch_cmd.rc != 0 - name: run batch --report to see if devices get filtered - command: "ceph-volume --cluster {{ cluster }} lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" + command: "ceph-volume lvm batch --report --format=json --{{ osd_objectstore|default('bluestore') }} {{ '--dmcrypt' if dmcrypt|default(false) else '' }} {{ devices[:2] | join(' ') }} {{ external_devices }} {{ devices[2:] | join(' ') }}" register: report_cmd failed_when: false environment: diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_zap.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_zap.yml index 9d63df9e0..4408288c8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_zap.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/test_zap.yml @@ -15,10 +15,10 @@ tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + command: "ceph osd down osd.{{ item }}" with_items: "{{ osd_ids }}" - name: purge osds - command: "ceph --cluster {{ cluster }} osd purge osd.{{ item }} --yes-i-really-mean-it" + command: "ceph osd purge osd.{{ item }} --yes-i-really-mean-it" with_items: "{{ osd_ids }}" @@ -27,7 +27,7 @@ tasks: - name: zap devices used for OSDs - command: "ceph-volume --cluster {{ cluster }} lvm zap --osd-id {{ item }} --destroy" + command: "ceph-volume lvm zap --osd-id {{ item }} --destroy" with_items: "{{ osd_ids }}" environment: CEPH_VOLUME_DEBUG: 1 diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini index 1d9c58d33..783142737 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = centos8-bluestore-{single_type,single_type_dmcrypt,mixed_type,mixed_type_dmcrypt,mixed_type_explicit,mixed_type_dmcrypt_explicit} +envlist = centos-bluestore-{single_type,single_type_dmcrypt,mixed_type,mixed_type_dmcrypt,mixed_type_explicit,mixed_type_dmcrypt_explicit} skipsdist = True [testenv] @@ -18,17 +18,19 @@ setenv= VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 DEBIAN_FRONTEND=noninteractive + ANSIBLE_COLLECTIONS_PATH = {envdir}/ansible_collections + CEPH_ANSIBLE_VAGRANT_BOX = centos/stream8 changedir= - centos8-bluestore-single_type: {toxinidir}/centos8/bluestore/single-type - centos8-bluestore-single_type_dmcrypt: {toxinidir}/centos8/bluestore/single-type-dmcrypt - centos8-bluestore-mixed_type: {toxinidir}/centos8/bluestore/mixed-type - centos8-bluestore-mixed_type_dmcrypt: {toxinidir}/centos8/bluestore/mixed-type-dmcrypt - centos8-bluestore-mixed_type_explicit: {toxinidir}/centos8/bluestore/mixed-type-explicit - centos8-bluestore-mixed_type_dmcrypt_explicit: {toxinidir}/centos8/bluestore/mixed-type-dmcrypt-explicit + centos-bluestore-single_type: {toxinidir}/centos/bluestore/single-type + centos-bluestore-single_type_dmcrypt: {toxinidir}/centos/bluestore/single-type-dmcrypt + centos-bluestore-mixed_type: {toxinidir}/centos/bluestore/mixed-type + centos-bluestore-mixed_type_dmcrypt: {toxinidir}/centos/bluestore/mixed-type-dmcrypt + centos-bluestore-mixed_type_explicit: {toxinidir}/centos/bluestore/mixed-type-explicit + centos-bluestore-mixed_type_dmcrypt_explicit: {toxinidir}/centos/bluestore/mixed-type-dmcrypt-explicit commands= git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible python -m pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt - ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v + ansible-galaxy collection install -r {envdir}/tmp/ceph-ansible/requirements.yml -v -p {envdir}/ansible_collections # bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:""} {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/vagrant_up.sh {posargs:--provider=virtualbox} @@ -44,9 +46,6 @@ commands= # use ceph-ansible to deploy a ceph cluster on the vms ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}" - # prepare nodes for testing with testinfra - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml - # test cluster state using testinfra py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests diff --git a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore index ca0146b19..1a4fadc10 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore +++ b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore @@ -1,10 +1,8 @@ --- ceph_dev: True -cluster: test public_network: "192.168.3.0/24" cluster_network: "192.168.4.0/24" -monitor_interface: eth1 osd_objectstore: "bluestore" osd_scenario: lvm num_osds: 2 diff --git a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm index c333af3e5..40abe4c8c 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm +++ b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm @@ -1,10 +1,8 @@ --- ceph_dev: True -cluster: test public_network: "192.168.3.0/24" cluster_network: "192.168.4.0/24" -monitor_interface: eth1 journal_size: 100 osd_objectstore: "bluestore" osd_scenario: lvm diff --git a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt index 3cd68aaf1..5f8eb3827 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt +++ b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_lvm_dmcrypt @@ -2,10 +2,8 @@ dmcrypt: True ceph_dev: True -cluster: test public_network: "192.168.3.0/24" cluster_network: "192.168.4.0/24" -monitor_interface: eth1 journal_size: 100 osd_objectstore: "bluestore" osd_scenario: lvm diff --git a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_single b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_single index e43b14a75..688d65352 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_single +++ b/src/ceph-volume/ceph_volume/tests/functional/group_vars/bluestore_single @@ -1,10 +1,8 @@ --- ceph_dev: True -cluster: test public_network: "192.168.3.0/24" cluster_network: "192.168.4.0/24" -monitor_interface: eth1 osd_objectstore: "bluestore" osd_scenario: lvm ceph_origin: 'repository' diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/group_vars/all index 5a7af3be0..5a7af3be0 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/hosts b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/setup.yml index 1c1a3ce8d..1c1a3ce8d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/test.yml index 165d9da29..165d9da29 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/test.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/create/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/create/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/Vagrantfile index 16076e424..16076e424 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/Vagrantfile diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/group_vars/all index 6ef6a9844..6ef6a9844 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/group_vars/all +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/group_vars/all diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/hosts b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/hosts index e1c1de6f8..e1c1de6f8 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/hosts +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/hosts diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/setup.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/setup.yml index 1c1a3ce8d..1c1a3ce8d 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/setup.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/setup.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/test.yml index 0a47b5eb8..c35591ca0 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/test.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/test.yml @@ -16,20 +16,20 @@ become: yes tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + command: "ceph osd down osd.{{ item }}" with_items: - 0 - 2 - name: destroy osd.2 - command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + command: "ceph osd destroy osd.2 --yes-i-really-mean-it" register: result retries: 30 delay: 1 until: result is succeeded - name: destroy osd.0 - command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" register: result retries: 30 delay: 1 @@ -41,7 +41,7 @@ # osd.2 device - name: zap /dev/vdd1 - command: "ceph-volume --cluster {{ cluster }} lvm zap /dev/vdd1 --destroy" + command: "ceph-volume lvm zap /dev/vdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 @@ -57,18 +57,18 @@ state: present - name: redeploy osd.2 using /dev/vdd1 - command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data /dev/vdd1 --osd-id 2" + command: "ceph-volume lvm create --bluestore --data /dev/vdd1 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 # osd.0 lv - name: zap test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1" + command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - name: redeploy osd.0 using test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data test_group/data-lv1 --osd-id 0" + command: "ceph-volume lvm create --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 @@ -82,10 +82,10 @@ become: yes tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.0" + command: "ceph osd down osd.0" - name: destroy osd.0 - command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" register: result retries: 30 delay: 1 @@ -98,12 +98,12 @@ - name: zap test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1" + command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - name: prepare osd.0 using test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" + command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/vagrant_variables.yml index d21531f6c..d21531f6c 120000 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/centos8/bluestore/dmcrypt/vagrant_variables.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/centos/bluestore/dmcrypt/vagrant_variables.yml diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml index 97d77a7f4..b6b038c90 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/playbooks/test_bluestore.yml @@ -18,20 +18,20 @@ become: yes tasks: - name: mark osds down - command: "ceph --cluster {{ cluster }} osd down osd.{{ item }}" + command: "ceph osd down osd.{{ item }}" with_items: - 0 - 2 - name: destroy osd.2 - command: "ceph --cluster {{ cluster }} osd destroy osd.2 --yes-i-really-mean-it" + command: "ceph osd destroy osd.2 --yes-i-really-mean-it" register: result retries: 30 delay: 1 until: result is succeeded - name: destroy osd.0 - command: "ceph --cluster {{ cluster }} osd destroy osd.0 --yes-i-really-mean-it" + command: "ceph osd destroy osd.0 --yes-i-really-mean-it" register: result retries: 30 delay: 1 @@ -44,7 +44,7 @@ # osd.2 device - name: zap /dev/vdd1 - command: "ceph-volume --cluster {{ cluster }} lvm zap /dev/vdd1 --destroy" + command: "ceph-volume lvm zap /dev/vdd1 --destroy" environment: CEPH_VOLUME_DEBUG: 1 @@ -60,18 +60,18 @@ state: present - name: redeploy osd.2 using /dev/vdd1 - command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data /dev/vdd1 --osd-id 2" + command: "ceph-volume lvm create --bluestore --data /dev/vdd1 --osd-id 2" environment: CEPH_VOLUME_DEBUG: 1 # osd.0 device (zap without --destroy that removes the LV) - name: zap test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm zap test_group/data-lv1" + command: "ceph-volume lvm zap test_group/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - name: prepare osd.0 again using test_group/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" + command: "ceph-volume lvm prepare --bluestore --data test_group/data-lv1 --osd-id 0" environment: CEPH_VOLUME_DEBUG: 1 @@ -151,11 +151,11 @@ # zapping the first lv shouldn't remove the vg, allowing the second zap to succeed - name: zap test_zap/data-lv1 - command: "ceph-volume --cluster {{ cluster }} lvm zap --destroy test_zap/data-lv1" + command: "ceph-volume lvm zap --destroy test_zap/data-lv1" environment: CEPH_VOLUME_DEBUG: 1 - name: zap test_zap/data-lv2 - command: "ceph-volume --cluster {{ cluster }} lvm zap --destroy test_zap/data-lv2" + command: "ceph-volume lvm zap --destroy test_zap/data-lv2" environment: CEPH_VOLUME_DEBUG: 1 diff --git a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini index 49c969059..8b49b7ac2 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini +++ b/src/ceph-volume/ceph_volume/tests/functional/lvm/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = centos8-bluestore-{create,prepare_activate,dmcrypt} +envlist = centos-bluestore-{create,prepare_activate,dmcrypt} skipsdist = True [testenv] @@ -18,18 +18,20 @@ setenv= VAGRANT_CWD = {changedir} CEPH_VOLUME_DEBUG = 1 DEBIAN_FRONTEND=noninteractive + ANSIBLE_COLLECTIONS_PATH = {envdir}/ansible_collections + CEPH_ANSIBLE_VAGRANT_BOX = centos/stream8 changedir= # plain/unencrypted - centos8-bluestore-create: {toxinidir}/centos8/bluestore/create + centos-bluestore-create: {toxinidir}/centos/bluestore/create # dmcrypt - centos8-bluestore-dmcrypt: {toxinidir}/centos8/bluestore/dmcrypt + centos-bluestore-dmcrypt: {toxinidir}/centos/bluestore/dmcrypt # TODO: these are placeholders for now, eventually we want to # test the prepare/activate workflow of ceph-volume as well - centos8-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate + centos-bluestore-prepare_activate: {toxinidir}/xenial/bluestore/prepare_activate commands= - git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible + git clone -b {env:CEPH_ANSIBLE_BRANCH:main} --single-branch {env:CEPH_ANSIBLE_CLONE:"https://github.com/ceph/ceph-ansible.git"} {envdir}/tmp/ceph-ansible pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt - ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v + ansible-galaxy collection install -r {envdir}/tmp/ceph-ansible/requirements.yml -v -p {envdir}/ansible_collections bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} @@ -45,10 +47,7 @@ commands= cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible # use ceph-ansible to deploy a ceph cluster on the vms - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}" - - # prepare nodes for testing with testinfra - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml + ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:main} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}" # test cluster state using testinfra py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests diff --git a/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml b/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml index 0ac200c6b..036c4daf5 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/playbooks/deploy.yml @@ -21,20 +21,6 @@ DEBIAN_FRONTEND: noninteractive pre_tasks: - # If we can't get python2 installed before any module is used we will fail - # so just try what we can to get it installed - - name: check for python2 - stat: - path: /usr/bin/python - ignore_errors: yes - register: systempython2 - - - name: install python2 for debian based systems - raw: sudo apt-get -y install python-simplejson - ignore_errors: yes - when: - - systempython2.stat is undefined or systempython2.stat.exists == false - # Ansible will try to auto-install python-apt, in some systems this might be # python3-apt, or python-apt, and it has caused whole runs to fail because # it is trying to do an interactive prompt @@ -46,18 +32,6 @@ - python-apt - aptitude - - name: install python2 for fedora - raw: sudo dnf -y install python creates=/usr/bin/python - ignore_errors: yes - when: - - systempython2.stat is undefined or systempython2.stat.exists == false - - - name: install python2 for opensuse - raw: sudo zypper -n install python-base creates=/usr/bin/python2.7 - ignore_errors: yes - when: - - systempython2.stat is undefined or systempython2.stat.exists == false - - name: gather facts setup: when: @@ -93,6 +67,12 @@ state: latest when: not is_atomic | bool + - name: install net-tools + package: + name: net-tools + state: present + when: not is_atomic | bool + - name: update the system command: dnf update -y changed_when: false diff --git a/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_up.sh b/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_up.sh index 8f4cd3bca..104ab118c 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_up.sh +++ b/src/ceph-volume/ceph_volume/tests/functional/scripts/vagrant_up.sh @@ -2,6 +2,15 @@ set -e +CEPH_ANSIBLE_VAGRANT_BOX="${CEPH_ANSIBLE_VAGRANT_BOX:-centos/stream9}" + +if [[ "${CEPH_ANSIBLE_VAGRANT_BOX}" =~ "centos/stream" ]]; then + EL_VERSION="${CEPH_ANSIBLE_VAGRANT_BOX: -1}" + LATEST_IMAGE="$(curl -s https://cloud.centos.org/centos/${EL_VERSION}-stream/x86_64/images/CHECKSUM | sed -nE 's/^SHA256.*\((.*-([0-9]+).*vagrant-libvirt.box)\).*$/\1/p' | sort -u | tail -n1)" + vagrant box remove "${CEPH_ANSIBLE_VAGRANT_BOX}" --all --force || true + vagrant box add --force --provider libvirt --name "${CEPH_ANSIBLE_VAGRANT_BOX}" "https://cloud.centos.org/centos/${EL_VERSION}-stream/x86_64/images/${LATEST_IMAGE}" --force +fi + retries=0 until [ $retries -ge 5 ] do diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile deleted file mode 120000 index 16076e424..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/Vagrantfile +++ /dev/null @@ -1 +0,0 @@ -../../../../Vagrantfile
\ No newline at end of file diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all deleted file mode 100644 index c265e783b..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/group_vars/all +++ /dev/null @@ -1,19 +0,0 @@ ---- - -ceph_dev: True -cluster: test -public_network: "192.168.1.0/24" -cluster_network: "192.168.2.0/24" -monitor_interface: eth1 -journal_size: 100 -osd_objectstore: "bluestore" -ceph_origin: 'repository' -ceph_repository: 'dev' -copy_admin_key: false -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } -ceph_conf_overrides: - global: - osd_pool_default_pg_num: 8 - osd_pool_default_size: 1 diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml deleted file mode 100644 index 2e1c7ee9e..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd0.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -devices: - - '/dev/sdb' -dedicated_devices: - - '/dev/sdc' -osd_scenario: "non-collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml deleted file mode 100644 index 7e90071c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/host_vars/osd1.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -devices: - - '/dev/sdb' - - '/dev/sdc' -osd_scenario: "collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts deleted file mode 100644 index e0c08b946..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/hosts +++ /dev/null @@ -1,9 +0,0 @@ -[mons] -mon0 monitor_interface=eth1 - -[osds] -osd0 -osd1 - -[mgrs] -mon0 diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml deleted file mode 100644 index 24e2c0353..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/test.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- hosts: osds - become: yes - tasks: - - - name: list all OSD directories - find: - paths: /var/lib/ceph/osd - file_type: directory - register: osd_paths - - - name: scan all OSD directories - command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" - environment: - CEPH_VOLUME_DEBUG: 1 - with_items: - - "{{ osd_paths.files }}" - - - name: list all OSD JSON files - find: - paths: /etc/ceph/osd - file_type: file - register: osd_configs - - - name: activate all scanned OSDs - command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" - environment: - CEPH_VOLUME_DEBUG: 1 - with_items: - - "{{ osd_configs.files }}" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml deleted file mode 100644 index 63700c3c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/activate/vagrant_variables.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -# DEPLOY CONTAINERIZED DAEMONS -docker: false - -# DEFINE THE NUMBER OF VMS TO RUN -mon_vms: 1 -osd_vms: 2 -mds_vms: 0 -rgw_vms: 0 -nfs_vms: 0 -rbd_mirror_vms: 0 -client_vms: 0 -iscsi_gw_vms: 0 -mgr_vms: 0 - - -# INSTALL SOURCE OF CEPH -# valid values are 'stable' and 'dev' -ceph_install_source: stable - -# SUBNETS TO USE FOR THE VMS -public_subnet: 192.168.1 -cluster_subnet: 192.168.2 - -# MEMORY -# set 1024 for CentOS -memory: 512 - -# Ethernet interface name -# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial -eth: 'eth1' - -# Disks -# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" -# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" -disks: "[ '/dev/sdb', '/dev/sdc' ]" - -# VAGRANT BOX -# Ceph boxes are *strongly* suggested. They are under better control and will -# not get updated frequently unless required for build systems. These are (for -# now): -# -# * ceph/ubuntu-xenial -# -# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 -# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet -# libvirt CentOS: centos/7 -# parallels Ubuntu: parallels/ubuntu-14.04 -# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' -# For more boxes have a look at: -# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= -# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ -vagrant_box: centos/7 -#ssh_private_key_path: "~/.ssh/id_rsa" -# The sync directory changes based on vagrant box -# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant -#vagrant_sync_dir: /home/vagrant/sync -#vagrant_sync_dir: / -# Disables synced folder creation. Not needed for testing, will skip mounting -# the vagrant directory on the remote box regardless of the provider. -vagrant_disable_synced_folder: true -# VAGRANT URL -# This is a URL to download an image from an alternate location. vagrant_box -# above should be set to the filename of the image. -# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box -# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box -# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box - -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } - diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/Vagrantfile deleted file mode 120000 index 16076e424..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/Vagrantfile +++ /dev/null @@ -1 +0,0 @@ -../../../../Vagrantfile
\ No newline at end of file diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all deleted file mode 100644 index 885c2c82f..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/group_vars/all +++ /dev/null @@ -1,22 +0,0 @@ ---- - -dmcrypt: True -ceph_dev: True -cluster: test -public_network: "192.168.1.0/24" -cluster_network: "192.168.2.0/24" -monitor_interface: eth1 -journal_size: 100 -osd_objectstore: "bluestore" -ceph_origin: 'repository' -ceph_repository: 'dev' -copy_admin_key: false -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } -ceph_conf_overrides: - global: - osd_pool_default_pg_num: 8 - osd_pool_default_size: 1 - osd: - osd_dmcrypt_type: luks diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd0.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd0.yml deleted file mode 100644 index 2e1c7ee9e..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd0.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -devices: - - '/dev/sdb' -dedicated_devices: - - '/dev/sdc' -osd_scenario: "non-collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd1.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd1.yml deleted file mode 100644 index 7e90071c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/host_vars/osd1.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -devices: - - '/dev/sdb' - - '/dev/sdc' -osd_scenario: "collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/hosts b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/hosts deleted file mode 100644 index e0c08b946..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/hosts +++ /dev/null @@ -1,9 +0,0 @@ -[mons] -mon0 monitor_interface=eth1 - -[osds] -osd0 -osd1 - -[mgrs] -mon0 diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/test.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/test.yml deleted file mode 100644 index 55ae7cc8e..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/test.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -- hosts: osds - become: yes - tasks: - - - name: scan all running OSDs - command: "ceph-volume --cluster={{ cluster }} simple scan" - environment: - CEPH_VOLUME_DEBUG: 1 - - - name: activate all scanned OSDs - command: "ceph-volume --cluster={{ cluster }} simple activate --all" - environment: - CEPH_VOLUME_DEBUG: 1 diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/vagrant_variables.yml deleted file mode 100644 index 63700c3c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-luks/vagrant_variables.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -# DEPLOY CONTAINERIZED DAEMONS -docker: false - -# DEFINE THE NUMBER OF VMS TO RUN -mon_vms: 1 -osd_vms: 2 -mds_vms: 0 -rgw_vms: 0 -nfs_vms: 0 -rbd_mirror_vms: 0 -client_vms: 0 -iscsi_gw_vms: 0 -mgr_vms: 0 - - -# INSTALL SOURCE OF CEPH -# valid values are 'stable' and 'dev' -ceph_install_source: stable - -# SUBNETS TO USE FOR THE VMS -public_subnet: 192.168.1 -cluster_subnet: 192.168.2 - -# MEMORY -# set 1024 for CentOS -memory: 512 - -# Ethernet interface name -# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial -eth: 'eth1' - -# Disks -# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" -# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" -disks: "[ '/dev/sdb', '/dev/sdc' ]" - -# VAGRANT BOX -# Ceph boxes are *strongly* suggested. They are under better control and will -# not get updated frequently unless required for build systems. These are (for -# now): -# -# * ceph/ubuntu-xenial -# -# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 -# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet -# libvirt CentOS: centos/7 -# parallels Ubuntu: parallels/ubuntu-14.04 -# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' -# For more boxes have a look at: -# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= -# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ -vagrant_box: centos/7 -#ssh_private_key_path: "~/.ssh/id_rsa" -# The sync directory changes based on vagrant box -# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant -#vagrant_sync_dir: /home/vagrant/sync -#vagrant_sync_dir: / -# Disables synced folder creation. Not needed for testing, will skip mounting -# the vagrant directory on the remote box regardless of the provider. -vagrant_disable_synced_folder: true -# VAGRANT URL -# This is a URL to download an image from an alternate location. vagrant_box -# above should be set to the filename of the image. -# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box -# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box -# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box - -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } - diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/Vagrantfile deleted file mode 120000 index 16076e424..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/Vagrantfile +++ /dev/null @@ -1 +0,0 @@ -../../../../Vagrantfile
\ No newline at end of file diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all deleted file mode 100644 index 30bcf5be7..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/group_vars/all +++ /dev/null @@ -1,22 +0,0 @@ ---- - -dmcrypt: True -ceph_dev: True -cluster: test -public_network: "192.168.1.0/24" -cluster_network: "192.168.2.0/24" -monitor_interface: eth1 -journal_size: 100 -osd_objectstore: "bluestore" -ceph_origin: 'repository' -ceph_repository: 'dev' -copy_admin_key: false -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } -ceph_conf_overrides: - global: - osd_pool_default_pg_num: 8 - osd_pool_default_size: 1 - osd: - osd_dmcrypt_type: plain diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd0.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd0.yml deleted file mode 100644 index 2e1c7ee9e..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd0.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -devices: - - '/dev/sdb' -dedicated_devices: - - '/dev/sdc' -osd_scenario: "non-collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd1.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd1.yml deleted file mode 100644 index 7e90071c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/host_vars/osd1.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -devices: - - '/dev/sdb' - - '/dev/sdc' -osd_scenario: "collocated" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/hosts b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/hosts deleted file mode 100644 index e0c08b946..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/hosts +++ /dev/null @@ -1,9 +0,0 @@ -[mons] -mon0 monitor_interface=eth1 - -[osds] -osd0 -osd1 - -[mgrs] -mon0 diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/test.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/test.yml deleted file mode 100644 index 24e2c0353..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/test.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- hosts: osds - become: yes - tasks: - - - name: list all OSD directories - find: - paths: /var/lib/ceph/osd - file_type: directory - register: osd_paths - - - name: scan all OSD directories - command: "ceph-volume --cluster={{ cluster }} simple scan {{ item.path }}" - environment: - CEPH_VOLUME_DEBUG: 1 - with_items: - - "{{ osd_paths.files }}" - - - name: list all OSD JSON files - find: - paths: /etc/ceph/osd - file_type: file - register: osd_configs - - - name: activate all scanned OSDs - command: "ceph-volume --cluster={{ cluster }} simple activate --file {{ item.path }}" - environment: - CEPH_VOLUME_DEBUG: 1 - with_items: - - "{{ osd_configs.files }}" diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/vagrant_variables.yml b/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/vagrant_variables.yml deleted file mode 100644 index 63700c3c9..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/centos7/bluestore/dmcrypt-plain/vagrant_variables.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -# DEPLOY CONTAINERIZED DAEMONS -docker: false - -# DEFINE THE NUMBER OF VMS TO RUN -mon_vms: 1 -osd_vms: 2 -mds_vms: 0 -rgw_vms: 0 -nfs_vms: 0 -rbd_mirror_vms: 0 -client_vms: 0 -iscsi_gw_vms: 0 -mgr_vms: 0 - - -# INSTALL SOURCE OF CEPH -# valid values are 'stable' and 'dev' -ceph_install_source: stable - -# SUBNETS TO USE FOR THE VMS -public_subnet: 192.168.1 -cluster_subnet: 192.168.2 - -# MEMORY -# set 1024 for CentOS -memory: 512 - -# Ethernet interface name -# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial -eth: 'eth1' - -# Disks -# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" -# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" -disks: "[ '/dev/sdb', '/dev/sdc' ]" - -# VAGRANT BOX -# Ceph boxes are *strongly* suggested. They are under better control and will -# not get updated frequently unless required for build systems. These are (for -# now): -# -# * ceph/ubuntu-xenial -# -# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 -# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet -# libvirt CentOS: centos/7 -# parallels Ubuntu: parallels/ubuntu-14.04 -# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' -# For more boxes have a look at: -# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= -# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ -vagrant_box: centos/7 -#ssh_private_key_path: "~/.ssh/id_rsa" -# The sync directory changes based on vagrant box -# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant -#vagrant_sync_dir: /home/vagrant/sync -#vagrant_sync_dir: / -# Disables synced folder creation. Not needed for testing, will skip mounting -# the vagrant directory on the remote box regardless of the provider. -vagrant_disable_synced_folder: true -# VAGRANT URL -# This is a URL to download an image from an alternate location. vagrant_box -# above should be set to the filename of the image. -# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box -# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box -# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box - -os_tuning_params: - - { name: kernel.pid_max, value: 4194303 } - - { name: fs.file-max, value: 26234859 } - diff --git a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini b/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini deleted file mode 100644 index c910754c3..000000000 --- a/src/ceph-volume/ceph_volume/tests/functional/simple/tox.ini +++ /dev/null @@ -1,56 +0,0 @@ -[tox] -envlist = centos7-bluestore-{activate,dmcrypt_plain,dmcrypt_luks} -skipsdist = True - -[testenv] -deps = mock -allowlist_externals = - vagrant - bash - git - sleep - cp -passenv=* -setenv= - ANSIBLE_CONFIG = {envdir}/tmp/ceph-ansible/ansible.cfg - ANSIBLE_SSH_ARGS = -F {changedir}/vagrant_ssh_config -o ControlMaster=auto -o ControlPersist=600s -o PreferredAuthentications=publickey - ANSIBLE_STDOUT_CALLBACK = debug - VAGRANT_CWD = {changedir} - CEPH_VOLUME_DEBUG = 1 - DEBIAN_FRONTEND=noninteractive -changedir= - centos7-bluestore-activate: {toxinidir}/centos7/bluestore/activate - centos7-bluestore-dmcrypt_plain: {toxinidir}/centos7/bluestore/dmcrypt-plain - centos7-bluestore-dmcrypt_luks: {toxinidir}/centos7/bluestore/dmcrypt-luks -commands= - git clone -b {env:CEPH_ANSIBLE_BRANCH:master} --single-branch https://github.com/ceph/ceph-ansible.git {envdir}/tmp/ceph-ansible - pip install -r {envdir}/tmp/ceph-ansible/tests/requirements.txt - ansible-galaxy install -r {envdir}/tmp/ceph-ansible/requirements.yml -v - - bash {toxinidir}/../scripts/vagrant_up.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} - bash {toxinidir}/../scripts/generate_ssh_config.sh {changedir} - - cp {toxinidir}/../playbooks/deploy.yml {envdir}/tmp/ceph-ansible - - # use ceph-ansible to deploy a ceph cluster on the vms - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/deploy.yml --extra-vars "fetch_directory={changedir}/fetch ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} toxinidir={toxinidir}" - - # prepare nodes for testing with testinfra - ansible-playbook -vv -i {changedir}/hosts {envdir}/tmp/ceph-ansible/tests/functional/setup.yml - - # test cluster state testinfra - py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests - - # make ceph-volume simple take over all the OSDs that got deployed, disabling ceph-disk - ansible-playbook -vv -i {changedir}/hosts {changedir}/test.yml - - # reboot all vms - bash {toxinidir}/../scripts/vagrant_reload.sh {env:VAGRANT_UP_FLAGS:"--no-provision"} {posargs:--provider=virtualbox} - - # wait 2 minutes for services to be ready - sleep 120 - - # retest to ensure cluster came back up correctly after rebooting - py.test --reruns 5 --reruns-delay 10 -n 4 --sudo -v --connection=ansible --ssh-config={changedir}/vagrant_ssh_config --ansible-inventory={changedir}/hosts {toxinidir}/../tests - - vagrant destroy {env:VAGRANT_DESTROY_FLAGS:"--force"} diff --git a/src/ceph-volume/ceph_volume/tests/util/test_device.py b/src/ceph-volume/ceph_volume/tests/util/test_device.py index e382981d9..69b57c211 100644 --- a/src/ceph-volume/ceph_volume/tests/util/test_device.py +++ b/src/ceph-volume/ceph_volume/tests/util/test_device.py @@ -153,14 +153,6 @@ class TestDevice(object): assert disk.is_device is True @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) - def test_is_partition(self, fake_call, device_info): - data = {"/dev/sda1": {"foo": "bar"}} - lsblk = {"TYPE": "part", "NAME": "sda1", "PKNAME": "sda"} - device_info(devices=data, lsblk=lsblk) - disk = device.Device("/dev/sda1") - assert disk.is_partition - - @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_mpath_device_is_device(self, fake_call, device_info): data = {"/dev/foo": {"foo": "bar"}} lsblk = {"TYPE": "mpath", "NAME": "foo"} @@ -241,7 +233,7 @@ class TestDevice(object): @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_reject_removable_device(self, fake_call, device_info): - data = {"/dev/sdb": {"removable": 1}} + data = {"/dev/sdb": {"removable": "1"}} lsblk = {"TYPE": "disk", "NAME": "sdb"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/sdb") @@ -249,7 +241,7 @@ class TestDevice(object): @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_reject_device_with_gpt_headers(self, fake_call, device_info): - data = {"/dev/sdb": {"removable": 0, "size": 5368709120}} + data = {"/dev/sdb": {"removable": "0", "size": 5368709120}} lsblk = {"TYPE": "disk", "NAME": "sdb"} blkid= {"PTTYPE": "gpt"} device_info( @@ -262,7 +254,7 @@ class TestDevice(object): @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_accept_non_removable_device(self, fake_call, device_info): - data = {"/dev/sdb": {"removable": 0, "size": 5368709120}} + data = {"/dev/sdb": {"removable": "0", "size": 5368709120}} lsblk = {"TYPE": "disk", "NAME": "sdb"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/sdb") @@ -286,7 +278,7 @@ class TestDevice(object): fake_call): m_os_path_islink.return_value = True m_os_path_realpath.return_value = '/dev/sdb' - data = {"/dev/sdb": {"ro": 0, "size": 5368709120}} + data = {"/dev/sdb": {"ro": "0", "size": 5368709120}} lsblk = {"TYPE": "disk"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/test_symlink") @@ -304,7 +296,7 @@ class TestDevice(object): fake_call): m_os_path_islink.return_value = True m_os_readlink.return_value = '/dev/dm-0' - data = {"/dev/mapper/mpatha": {"ro": 0, "size": 5368709120}} + data = {"/dev/mapper/mpatha": {"ro": "0", "size": 5368709120}} lsblk = {"TYPE": "disk"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/mapper/mpatha") @@ -312,7 +304,7 @@ class TestDevice(object): @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_reject_readonly_device(self, fake_call, device_info): - data = {"/dev/cdrom": {"ro": 1}} + data = {"/dev/cdrom": {"ro": "1"}} lsblk = {"TYPE": "disk", "NAME": "cdrom"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/cdrom") @@ -328,7 +320,7 @@ class TestDevice(object): @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) def test_accept_non_readonly_device(self, fake_call, device_info): - data = {"/dev/sda": {"ro": 0, "size": 5368709120}} + data = {"/dev/sda": {"ro": "0", "size": 5368709120}} lsblk = {"TYPE": "disk", "NAME": "sda"} device_info(devices=data,lsblk=lsblk) disk = device.Device("/dev/sda") @@ -594,10 +586,10 @@ class TestDeviceOrdering(object): def setup_method(self): self.data = { - "/dev/sda": {"removable": 0}, - "/dev/sdb": {"removable": 1}, # invalid - "/dev/sdc": {"removable": 0}, - "/dev/sdd": {"removable": 1}, # invalid + "/dev/sda": {"removable": "0"}, + "/dev/sdb": {"removable": "1"}, # invalid + "/dev/sdc": {"removable": "0"}, + "/dev/sdd": {"removable": "1"}, # invalid } @patch("ceph_volume.util.disk.has_bluestore_label", lambda x: False) diff --git a/src/ceph-volume/ceph_volume/tests/util/test_disk.py b/src/ceph-volume/ceph_volume/tests/util/test_disk.py index ce1f9466f..b287530dc 100644 --- a/src/ceph-volume/ceph_volume/tests/util/test_disk.py +++ b/src/ceph-volume/ceph_volume/tests/util/test_disk.py @@ -33,6 +33,11 @@ class TestFunctions: def test_is_device_type_part(self): assert not disk.is_device('/dev/foo1') + @patch('ceph_volume.util.disk.os.path.exists', MagicMock(return_value=True)) + @patch('ceph_volume.util.disk.get_partitions', MagicMock(return_value={"sda1": "sda"})) + def test_is_partition(self): + assert disk.is_partition('sda1') + class TestLsblkParser(object): diff --git a/src/ceph-volume/ceph_volume/util/arg_validators.py b/src/ceph-volume/ceph_volume/util/arg_validators.py index 1abb5165e..e936cab89 100644 --- a/src/ceph-volume/ceph_volume/util/arg_validators.py +++ b/src/ceph-volume/ceph_volume/util/arg_validators.py @@ -4,11 +4,20 @@ import math from ceph_volume import terminal, decorators, process from ceph_volume.util.device import Device from ceph_volume.util import disk - +from ceph_volume.util.encryption import set_dmcrypt_no_workqueue +from ceph_volume import process, conf def valid_osd_id(val): return str(int(val)) +class DmcryptAction(argparse._StoreTrueAction): + def __init__(self, *args, **kwargs): + super(DmcryptAction, self).__init__(*args, **kwargs) + + def __call__(self, *args, **kwargs): + set_dmcrypt_no_workqueue() + super(DmcryptAction, self).__call__(*args, **kwargs) + class ValidDevice(object): def __init__(self, as_string=False, gpt_ok=False): diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py index d61222afe..1b52774d1 100644 --- a/src/ceph-volume/ceph_volume/util/device.py +++ b/src/ceph-volume/ceph_volume/util/device.py @@ -121,13 +121,8 @@ class Device(object): # check if we are not a device mapper if "dm-" not in real_path: self.path = real_path - if not sys_info.devices: - if self.path: - sys_info.devices = disk.get_devices(device=self.path) - else: - sys_info.devices = disk.get_devices() - if sys_info.devices.get(self.path, {}): - self.device_nodes = sys_info.devices[self.path]['device_nodes'] + if not sys_info.devices.get(self.path): + sys_info.devices = disk.get_devices() self.sys_api = sys_info.devices.get(self.path, {}) self.partitions = self._get_partitions() self.lv_api = None @@ -143,6 +138,8 @@ class Device(object): self._is_lvm_member = None self.ceph_device = False self._parse() + if self.path in sys_info.devices.keys(): + self.device_nodes = sys_info.devices[self.path]['device_nodes'] self.lsm_data = self.fetch_lsm(with_lsm) self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons() @@ -460,11 +457,11 @@ class Device(object): def device_type(self): self.load_blkid_api() if 'type' in self.sys_api: - return self.sys_api['type'] + return self.sys_api.get('type') elif self.disk_api: - return self.disk_api['TYPE'] + return self.disk_api.get('TYPE') elif self.blkid_api: - return self.blkid_api['TYPE'] + return self.blkid_api.get('TYPE') @property def is_mpath(self): @@ -478,9 +475,9 @@ class Device(object): def is_partition(self): self.load_blkid_api() if self.disk_api: - return self.disk_api['TYPE'] == 'part' + return self.disk_api.get('TYPE') == 'part' elif self.blkid_api: - return self.blkid_api['TYPE'] == 'part' + return self.blkid_api.get('TYPE') == 'part' return False @property @@ -594,8 +591,8 @@ class Device(object): def _check_generic_reject_reasons(self): reasons = [ - ('removable', 1, 'removable'), - ('ro', 1, 'read-only'), + ('id_bus', 'usb', 'id_bus'), + ('ro', '1', 'read-only'), ] rejected = [reason for (k, v, reason) in reasons if self.sys_api.get(k, '') == v] diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py index ee061b724..2984c391d 100644 --- a/src/ceph-volume/ceph_volume/util/disk.py +++ b/src/ceph-volume/ceph_volume/util/disk.py @@ -6,6 +6,7 @@ import time from ceph_volume import process from ceph_volume.api import lvm from ceph_volume.util.system import get_file_contents +from typing import Dict, List, Any logger = logging.getLogger(__name__) @@ -364,30 +365,18 @@ def is_device(dev): return TYPE in ['disk', 'mpath'] # fallback to stat - return _stat_is_device(os.lstat(dev).st_mode) + return _stat_is_device(os.lstat(dev).st_mode) and not is_partition(dev) -def is_partition(dev): +def is_partition(dev: str) -> bool: """ Boolean to determine if a given device is a partition, like /dev/sda1 """ if not os.path.exists(dev): return False - # use lsblk first, fall back to using stat - TYPE = lsblk(dev).get('TYPE') - if TYPE: - return TYPE == 'part' - - # fallback to stat - stat_obj = os.stat(dev) - if _stat_is_device(stat_obj.st_mode): - return False - major = os.major(stat_obj.st_rdev) - minor = os.minor(stat_obj.st_rdev) - if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): - return True - return False + partitions = get_partitions() + return dev.split("/")[-1] in partitions def is_ceph_rbd(dev): @@ -763,36 +752,34 @@ class AllowLoopDevices(object): allow_loop_devices = AllowLoopDevices() -def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block', device=''): - def holder_inner_loop(): +def get_block_devs_sysfs(_sys_block_path: str = '/sys/block', _sys_dev_block_path: str = '/sys/dev/block', device: str = '') -> List[List[str]]: + def holder_inner_loop() -> bool: for holder in holders: # /sys/block/sdy/holders/dm-8/dm/uuid - holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower() + holder_dm_type: str = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower() if holder_dm_type == 'mpath': return True # First, get devices that are _not_ partitions - result = list() + result: List[List[str]] = list() if not device: - dev_names = os.listdir(_sys_block_path) + dev_names: List[str] = os.listdir(_sys_block_path) else: dev_names = [device] for dev in dev_names: - name = kname = os.path.join("/dev", dev) + name = kname = pname = os.path.join("/dev", dev) if not os.path.exists(name): continue - type_ = 'disk' - holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders')) - if get_file_contents(os.path.join(_sys_block_path, dev, 'removable')) == "1": - continue + type_: str = 'disk' + holders: List[str] = os.listdir(os.path.join(_sys_block_path, dev, 'holders')) if holder_inner_loop(): continue - dm_dir_path = os.path.join(_sys_block_path, dev, 'dm') + dm_dir_path: str = os.path.join(_sys_block_path, dev, 'dm') if os.path.isdir(dm_dir_path): - dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid')) - type_ = dm_type.split('-')[0].lower() - basename = get_file_contents(os.path.join(dm_dir_path, 'name')) - name = os.path.join("/dev/mapper", basename) + dm_type: str = get_file_contents(os.path.join(dm_dir_path, 'uuid')) + type_: List[str] = dm_type.split('-')[0].lower() + basename: str = get_file_contents(os.path.join(dm_dir_path, 'name')) + name: str = os.path.join("/dev/mapper", basename) if dev.startswith('loop'): if not allow_loop_devices(): continue @@ -800,28 +787,25 @@ def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')): continue type_ = 'loop' - result.append([kname, name, type_]) + result.append([kname, name, type_, pname]) # Next, look for devices that _are_ partitions - for item in os.listdir(_sys_dev_block_path): - is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1" - dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item))) - if not is_part: - continue - name = kname = os.path.join("/dev", dev) - result.append([name, kname, "part"]) + partitions: Dict[str, str] = get_partitions() + for partition in partitions.keys(): + name = kname = os.path.join("/dev", partition) + result.append([name, kname, "part", partitions[partition]]) return sorted(result, key=lambda x: x[0]) -def get_partitions(_sys_dev_block_path ='/sys/dev/block'): - devices = os.listdir(_sys_dev_block_path) - result = dict() +def get_partitions(_sys_dev_block_path ='/sys/dev/block') -> List[str]: + devices: List[str] = os.listdir(_sys_dev_block_path) + result: Dict[str, str] = dict() for device in devices: - device_path = os.path.join(_sys_dev_block_path, device) - is_partition = get_file_contents(os.path.join(device_path, 'partition')) == "1" + device_path: str = os.path.join(_sys_dev_block_path, device) + is_partition: bool = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0 if not is_partition: continue - partition_sys_name = os.path.basename(os.readlink(device_path)) - parent_device_sys_name = os.readlink(device_path).split('/')[-2:-1][0] + partition_sys_name: str = os.path.basename(os.path.realpath(device_path)) + parent_device_sys_name: str = os.path.realpath(device_path).split('/')[-2:-1][0] result[partition_sys_name] = parent_device_sys_name return result @@ -839,13 +823,13 @@ def get_devices(_sys_block_path='/sys/block', device=''): device_facts = {} block_devs = get_block_devs_sysfs(_sys_block_path) - partitions = get_partitions() block_types = ['disk', 'mpath', 'lvm', 'part'] if allow_loop_devices(): block_types.append('loop') for block in block_devs: + metadata: Dict[str, Any] = {} if block[2] == 'lvm': block[1] = lvm.get_lv_path_from_mapper(block[1]) devname = os.path.basename(block[0]) @@ -854,8 +838,7 @@ def get_devices(_sys_block_path='/sys/block', device=''): continue sysdir = os.path.join(_sys_block_path, devname) if block[2] == 'part': - sysdir = os.path.join(_sys_block_path, partitions[devname], devname) - metadata = {} + sysdir = os.path.join(_sys_block_path, block[3], devname) # If the device is ceph rbd it gets excluded if is_ceph_rbd(diskname): @@ -890,7 +873,7 @@ def get_devices(_sys_block_path='/sys/block', device=''): metadata['device_nodes'] = ','.join(device_slaves) else: if block[2] == 'part': - metadata['device_nodes'] = partitions[devname] + metadata['device_nodes'] = block[3] else: metadata['device_nodes'] = devname @@ -920,7 +903,13 @@ def get_devices(_sys_block_path='/sys/block', device=''): metadata['size'] = float(size) * 512 metadata['human_readable_size'] = human_readable_size(metadata['size']) metadata['path'] = diskname + metadata['devname'] = devname metadata['type'] = block[2] + metadata['parent'] = block[3] + + # some facts from udevadm + p = udevadm_property(sysdir) + metadata['id_bus'] = p.get('ID_BUS', '') device_facts[diskname] = metadata return device_facts diff --git a/src/ceph-volume/ceph_volume/util/encryption.py b/src/ceph-volume/ceph_volume/util/encryption.py index f8aea80b4..844a81620 100644 --- a/src/ceph-volume/ceph_volume/util/encryption.py +++ b/src/ceph-volume/ceph_volume/util/encryption.py @@ -6,10 +6,28 @@ from ceph_volume.util import constants, system from ceph_volume.util.device import Device from .prepare import write_keyring from .disk import lsblk, device_family, get_part_entry_type +from packaging import version logger = logging.getLogger(__name__) mlogger = terminal.MultiLogger(__name__) +def set_dmcrypt_no_workqueue(target_version: str = '2.3.4') -> None: + """ + set `conf.dmcrypt_no_workqueue` to `True` if the available + version of `cryptsetup` is greater or equal to `version` + """ + command = ["cryptsetup", "--version"] + out, err, rc = process.call(command) + try: + if version.parse(out[0]) >= version.parse(f'cryptsetup {target_version}'): + conf.dmcrypt_no_workqueue = True + except IndexError: + mlogger.debug(f'cryptsetup version check: rc={rc} out={out} err={err}') + raise RuntimeError("Couldn't check the cryptsetup version.") + +def bypass_workqueue(device: str) -> bool: + return not Device(device).rotational and conf.dmcrypt_no_workqueue + def get_key_size_from_conf(): """ Return the osd dmcrypt key size from config file. @@ -79,6 +97,10 @@ def plain_open(key, device, mapping): '--key-size', '256', ] + if bypass_workqueue(device): + command.extend(['--perf-no_read_workqueue', + '--perf-no_write_workqueue']) + process.call(command, stdin=key, terminal_verbose=True, show_command=True) @@ -103,6 +125,11 @@ def luks_open(key, device, mapping): device, mapping, ] + + if bypass_workqueue(device): + command.extend(['--perf-no_read_workqueue', + '--perf-no_write_workqueue']) + process.call(command, stdin=key, terminal_verbose=True, show_command=True) |