summaryrefslogtreecommitdiffstats
path: root/qa/suites/orch/cephadm
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
commite6918187568dbd01842d8d1d2c808ce16a894239 (patch)
tree64f88b554b444a49f656b6c656111a145cbbaa28 /qa/suites/orch/cephadm
parentInitial commit. (diff)
downloadceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz
ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'qa/suites/orch/cephadm')
l---------qa/suites/orch/cephadm/.qa1
l---------qa/suites/orch/cephadm/mds_upgrade_sequence1
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/%0
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa1
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml1
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/.qa1
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.0.yaml8
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml8
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml6
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml29
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml29
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml43
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml10
l---------qa/suites/orch/cephadm/nfs1
-rw-r--r--qa/suites/orch/cephadm/orchestrator_cli/%0
l---------qa/suites/orch/cephadm/orchestrator_cli/.qa1
l---------qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$1
l---------qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml1
l---------qa/suites/orch/cephadm/orchestrator_cli/agent1
-rw-r--r--qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml19
-rw-r--r--qa/suites/orch/cephadm/osds/%0
l---------qa/suites/orch/cephadm/osds/.qa1
l---------qa/suites/orch/cephadm/osds/0-distro1
l---------qa/suites/orch/cephadm/osds/0-nvme-loop.yaml1
-rw-r--r--qa/suites/orch/cephadm/osds/1-start.yaml25
l---------qa/suites/orch/cephadm/osds/2-ops/.qa1
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml13
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml17
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml15
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml16
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml20
l---------qa/suites/orch/cephadm/rbd_iscsi1
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/%0
l---------qa/suites/orch/cephadm/smoke-roleless/.qa1
l---------qa/suites/orch/cephadm/smoke-roleless/0-distro1
l---------qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml1
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/1-start.yaml24
l---------qa/suites/orch/cephadm/smoke-roleless/2-services/.qa1
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml0
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml40
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml8
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/jaeger.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml9
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-haproxy-proto.yaml35
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml89
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml90
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml68
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml70
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-keepalive-only.yaml55
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml13
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nvmeof.yaml8
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml60
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/3-final.yaml10
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/%0
l---------qa/suites/orch/cephadm/smoke-singlehost/.qa1
l---------qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$1
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml27
l---------qa/suites/orch/cephadm/smoke-singlehost/2-services/.qa1
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml0
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml8
-rw-r--r--qa/suites/orch/cephadm/smoke-small/%0
l---------qa/suites/orch/cephadm/smoke-small/.qa1
l---------qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml1
l---------qa/suites/orch/cephadm/smoke-small/0-nvme-loop.yaml1
l---------qa/suites/orch/cephadm/smoke-small/agent/.qa1
-rw-r--r--qa/suites/orch/cephadm/smoke-small/agent/off.yaml5
-rw-r--r--qa/suites/orch/cephadm/smoke-small/agent/on.yaml5
-rw-r--r--qa/suites/orch/cephadm/smoke-small/fixed-2.yaml29
l---------qa/suites/orch/cephadm/smoke-small/mon_election1
-rw-r--r--qa/suites/orch/cephadm/smoke-small/start.yaml16
-rw-r--r--qa/suites/orch/cephadm/smoke/%0
l---------qa/suites/orch/cephadm/smoke/.qa1
l---------qa/suites/orch/cephadm/smoke/0-distro1
l---------qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml1
l---------qa/suites/orch/cephadm/smoke/agent/.qa1
-rw-r--r--qa/suites/orch/cephadm/smoke/agent/off.yaml5
-rw-r--r--qa/suites/orch/cephadm/smoke/agent/on.yaml5
-rw-r--r--qa/suites/orch/cephadm/smoke/fixed-2.yaml32
l---------qa/suites/orch/cephadm/smoke/mon_election1
-rw-r--r--qa/suites/orch/cephadm/smoke/start.yaml16
-rw-r--r--qa/suites/orch/cephadm/thrash/%0
l---------qa/suites/orch/cephadm/thrash/.qa1
l---------qa/suites/orch/cephadm/thrash/0-distro1
-rw-r--r--qa/suites/orch/cephadm/thrash/1-start.yaml7
-rw-r--r--qa/suites/orch/cephadm/thrash/2-thrash.yaml26
l---------qa/suites/orch/cephadm/thrash/3-tasks/.qa1
l---------qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml1
l---------qa/suites/orch/cephadm/thrash/fixed-2.yaml1
l---------qa/suites/orch/cephadm/thrash/msgr1
-rw-r--r--qa/suites/orch/cephadm/thrash/root.yaml3
-rw-r--r--qa/suites/orch/cephadm/upgrade/%0
l---------qa/suites/orch/cephadm/upgrade/.qa1
l---------qa/suites/orch/cephadm/upgrade/1-start-distro/.qa1
-rw-r--r--qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml39
-rw-r--r--qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml33
l---------qa/suites/orch/cephadm/upgrade/2-repo_digest/.qa1
-rw-r--r--qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml0
-rw-r--r--qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml4
l---------qa/suites/orch/cephadm/upgrade/3-upgrade/.qa1
-rw-r--r--qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml21
-rw-r--r--qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml132
-rw-r--r--qa/suites/orch/cephadm/upgrade/4-wait.yaml16
-rw-r--r--qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml6
l---------qa/suites/orch/cephadm/upgrade/agent1
l---------qa/suites/orch/cephadm/upgrade/mon_election1
-rw-r--r--qa/suites/orch/cephadm/with-work/%0
l---------qa/suites/orch/cephadm/with-work/.qa1
l---------qa/suites/orch/cephadm/with-work/0-distro1
l---------qa/suites/orch/cephadm/with-work/fixed-2.yaml1
l---------qa/suites/orch/cephadm/with-work/mode/.qa1
-rw-r--r--qa/suites/orch/cephadm/with-work/mode/packaged.yaml5
-rw-r--r--qa/suites/orch/cephadm/with-work/mode/root.yaml3
l---------qa/suites/orch/cephadm/with-work/mon_election1
l---------qa/suites/orch/cephadm/with-work/msgr1
-rw-r--r--qa/suites/orch/cephadm/with-work/start.yaml7
l---------qa/suites/orch/cephadm/with-work/tasks/.qa1
l---------qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml1
l---------qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml1
-rw-r--r--qa/suites/orch/cephadm/with-work/tasks/rotate-keys.yaml16
-rw-r--r--qa/suites/orch/cephadm/workunits/%0
l---------qa/suites/orch/cephadm/workunits/.qa1
l---------qa/suites/orch/cephadm/workunits/0-distro1
l---------qa/suites/orch/cephadm/workunits/agent1
l---------qa/suites/orch/cephadm/workunits/mon_election1
l---------qa/suites/orch/cephadm/workunits/task/.qa1
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_adoption.yaml11
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml31
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml11
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml8
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_iscsi_container/+0
l---------qa/suites/orch/cephadm/workunits/task/test_iscsi_container/.qa1
l---------qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml1
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml21
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml17
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml45
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml40
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml62
143 files changed, 1660 insertions, 0 deletions
diff --git a/qa/suites/orch/cephadm/.qa b/qa/suites/orch/cephadm/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mds_upgrade_sequence b/qa/suites/orch/cephadm/mds_upgrade_sequence
new file mode 120000
index 000000000..24aa41c10
--- /dev/null
+++ b/qa/suites/orch/cephadm/mds_upgrade_sequence
@@ -0,0 +1 @@
+.qa/suites/fs/upgrade/mds_upgrade_sequence/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/% b/qa/suites/orch/cephadm/mgr-nfs-upgrade/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/%
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml
new file mode 120000
index 000000000..7a86f967f
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-centos_8.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.stream_container_tools.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/.qa b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.0.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.0.yaml
new file mode 100644
index 000000000..beba37428
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.0.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: quay.io/ceph/ceph:v16.2.0
+ cephadm_branch: v16.2.0
+ cephadm_git_url: https://github.com/ceph/ceph
+ # needed for v16.2.0 due to --skip-admin-label
+ avoid_pacific_features: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
new file mode 100644
index 000000000..1cbe5a134
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: quay.io/ceph/ceph:v16.2.4
+ cephadm_branch: v16.2.4
+ cephadm_git_url: https://github.com/ceph/ceph
+ # needed for v16.2.4 due to --skip-admin-label
+ avoid_pacific_features: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
new file mode 100644
index 000000000..381088d5b
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
@@ -0,0 +1,6 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: quay.io/ceph/ceph:v16.2.5
+ cephadm_branch: v16.2.5
+ cephadm_git_url: https://github.com/ceph/ceph
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
new file mode 100644
index 000000000..2d9f09a4e
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
@@ -0,0 +1,29 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+- - host.b
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
new file mode 100644
index 000000000..34680fc8a
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
@@ -0,0 +1,29 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+
+- cephadm.wait_for_service:
+ service: mds.foofs
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2
+ - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+ # we can't do wait_for_service here because with octopus it's nfs.ganesha-foo not nfs.foo
+ - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done
+ - echo test > /mnt/foo/testfile
+ - sync
+
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
new file mode 100644
index 000000000..ec901a92e
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
@@ -0,0 +1,43 @@
+tasks:
+- parallel:
+ - upgrade-tasks
+ - workload-tasks
+
+upgrade-tasks:
+ sequential:
+ - cephadm.shell:
+ env: [sha1]
+ host.a:
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph config set global log_to_journald false --force
+ - ceph mgr module enable nfs --force
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
+ - cephadm.shell:
+ env: [sha1]
+ host.a:
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions
+ - echo "wait for servicemap items w/ changing names to refresh"
+ - sleep 60
+ - ceph orch ps
+ - ceph orch upgrade status
+ - ceph health detail
+ - ceph versions
+ - ceph versions | jq -e '.overall | length == 1'
+ - ceph versions | jq -e '.overall | keys' | grep $sha1
+
+ # this should be a no-op, but confirms nfs.ganesha-foo was remapped to nfs.foo
+ - cephadm.wait_for_service:
+ service: nfs.foo
+
+workload-tasks:
+ sequential:
+ - exec:
+ host.a:
+ - cd /mnt/foo && dbench 5 -t 600 || true # might fail with ESTALE
+ # make sure mount works
+ - umount /mnt/foo
+ - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done
+ - cd /mnt/foo && dbench 5 -t 5
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
new file mode 100644
index 000000000..3a9169659
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
@@ -0,0 +1,10 @@
+tasks:
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster ls | grep foo
+ - ceph nfs export ls foo --detailed
+ - rados -p .nfs --all ls -
+ - ceph config get mgr mgr/cephadm/migration_current | grep 6
diff --git a/qa/suites/orch/cephadm/nfs b/qa/suites/orch/cephadm/nfs
new file mode 120000
index 000000000..628e2a2a2
--- /dev/null
+++ b/qa/suites/orch/cephadm/nfs
@@ -0,0 +1 @@
+.qa/suites/fs/nfs/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/% b/qa/suites/orch/cephadm/orchestrator_cli/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/%
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/.qa b/qa/suites/orch/cephadm/orchestrator_cli/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml b/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml
new file mode 120000
index 000000000..8a0b9123b
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml
@@ -0,0 +1 @@
+.qa/clusters/2-node-mgr.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/agent b/qa/suites/orch/cephadm/orchestrator_cli/agent
new file mode 120000
index 000000000..154924209
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/agent
@@ -0,0 +1 @@
+../smoke/agent \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
new file mode 100644
index 000000000..3e6e7f955
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
@@ -0,0 +1,19 @@
+
+tasks:
+ - install:
+ - ceph:
+ # tests may leave mgrs broken, so don't try and call into them
+ # to invoke e.g. pg dump during teardown.
+ wait-for-scrub: false
+ log-ignorelist:
+ - overall HEALTH_
+ - \(MGR_DOWN\)
+ - \(DEVICE_IDENT_ON\)
+ - \(DEVICE_FAULT_ON\)
+ - \(PG_
+ - replacing it with standby
+ - No standby daemons available
+ - \(POOL_APP_NOT_ENABLED\)
+ - cephfs_test_runner:
+ modules:
+ - tasks.mgr.test_orchestrator_cli
diff --git a/qa/suites/orch/cephadm/osds/% b/qa/suites/orch/cephadm/osds/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/%
diff --git a/qa/suites/orch/cephadm/osds/.qa b/qa/suites/orch/cephadm/osds/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/0-distro b/qa/suites/orch/cephadm/osds/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml b/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/1-start.yaml b/qa/suites/orch/cephadm/osds/1-start.yaml
new file mode 100644
index 000000000..4331d7c66
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/1-start.yaml
@@ -0,0 +1,25 @@
+tasks:
+- cephadm:
+ roleless: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls | grep '^osd.all-available-devices '
+roles:
+- - host.a
+ - client.0
+- - host.b
+ - client.1
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/osds/2-ops/.qa b/qa/suites/orch/cephadm/osds/2-ops/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml
new file mode 100644
index 000000000..16413aba8
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml
@@ -0,0 +1,13 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ ceph osd tree
+ for osd in `ceph osd ls` ; do
+ ceph orch osd rm $osd --force --zap --replace
+ done
+ while ceph orch osd rm ls | wc | grep ^1 ; do sleep 10 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml
new file mode 100644
index 000000000..09be72f11
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml
@@ -0,0 +1,17 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep $DEVID | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep $DEVID | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ ceph orch device zap $HOST $DEV --force
+ ceph orch daemon add osd $HOST:$DEV
+ while ! ceph osd dump | grep osd.1 | grep up ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml
new file mode 100644
index 000000000..8f07f6d53
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml
@@ -0,0 +1,15 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep "$DEVID" | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep "$DEVID" | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm --zap --replace 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ while ! ceph osd dump | grep osd.1 | grep "up\s*in" ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml
new file mode 100644
index 000000000..78161aa49
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep $DEVID | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep $DEVID | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ ceph orch device zap $HOST $DEV --force
+ while ! ceph osd dump | grep osd.1 | grep up ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml b/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml
new file mode 100644
index 000000000..a971a02e4
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml
@@ -0,0 +1,20 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ HOST=$(hostname -s)
+ OSD=$(ceph orch ps $HOST | grep osd | head -n 1 | awk '{print $1}')
+ echo "host $HOST, osd $OSD"
+ ceph orch daemon stop $OSD
+ while ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done
+ ceph auth export $OSD > k
+ ceph orch daemon rm $OSD --force
+ ceph orch ps --refresh
+ while ceph orch ps | grep $OSD ; do sleep 5 ; done
+ ceph auth add $OSD -i k
+ ceph cephadm osd activate $HOST
+ while ! ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done
+- cephadm.healthy:
diff --git a/qa/suites/orch/cephadm/rbd_iscsi b/qa/suites/orch/cephadm/rbd_iscsi
new file mode 120000
index 000000000..f0073a119
--- /dev/null
+++ b/qa/suites/orch/cephadm/rbd_iscsi
@@ -0,0 +1 @@
+.qa/suites/rbd/iscsi \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/% b/qa/suites/orch/cephadm/smoke-roleless/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/%
diff --git a/qa/suites/orch/cephadm/smoke-roleless/.qa b/qa/suites/orch/cephadm/smoke-roleless/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/0-distro b/qa/suites/orch/cephadm/smoke-roleless/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml b/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml b/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml
new file mode 100644
index 000000000..018356f8f
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml
@@ -0,0 +1,24 @@
+tasks:
+- cephadm:
+ roleless: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - client.0
+- - host.b
+ - client.1
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/.qa b/qa/suites/orch/cephadm/smoke-roleless/2-services/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml
new file mode 100644
index 000000000..f00800471
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml
@@ -0,0 +1,40 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch host label add `hostname` foo
+ - ceph auth get-or-create client.foo mon 'allow r'
+ - ceph orch client-keyring set client.foo label:foo --mode 770 --owner 11111:22222
+- exec:
+ host.a:
+ - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx---
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222
+ - test -e /etc/ceph/ceph.conf
+- exec:
+ host.b:
+ - test ! -e /etc/ceph/ceph.client.foo.keyring
+- cephadm.shell:
+ host.b:
+ - ceph orch host label add `hostname` foo
+- exec:
+ host.b:
+ - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx---
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222
+- cephadm.shell:
+ host.b:
+ - ceph orch host label rm `hostname` foo
+- exec:
+ host.b:
+ - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+- exec:
+ host.a:
+ - test -e /etc/ceph/ceph.client.foo.keyring
+- cephadm.shell:
+ host.a:
+ - ceph orch client-keyring rm client.foo
+- exec:
+ host.a:
+ - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml
new file mode 100644
index 000000000..7f57076db
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply iscsi foo u p
+- cephadm.wait_for_service:
+ service: iscsi.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/jaeger.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/jaeger.yaml
new file mode 100644
index 000000000..ad102fedd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/jaeger.yaml
@@ -0,0 +1,12 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply jaeger
+- cephadm.wait_for_service:
+ service: elasticsearch
+- cephadm.wait_for_service:
+ service: jaeger-collector
+- cephadm.wait_for_service:
+ service: jaeger-query
+- cephadm.wait_for_service:
+ service: jaeger-agent \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml
new file mode 100644
index 000000000..681e1e04a
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml
@@ -0,0 +1,9 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rbd-mirror "--placement=*"
+ - ceph orch apply cephfs-mirror "--placement=*"
+- cephadm.wait_for_service:
+ service: rbd-mirror
+- cephadm.wait_for_service:
+ service: cephfs-mirror
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-haproxy-proto.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-haproxy-proto.yaml
new file mode 100644
index 000000000..477e5c443
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-haproxy-proto.yaml
@@ -0,0 +1,35 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+# use nfs module to create cluster and export
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create fs1
+ - ceph nfs cluster create happy --ingress --virtual-ip={{VIP0}} --ingress-mode=haproxy-protocol
+ - ceph nfs export create cephfs --fsname fs1 --cluster-id happy --pseudo-path /d1
+
+# wait for services to start
+- cephadm.wait_for_service:
+ service: nfs.happy
+- cephadm.wait_for_service:
+ service: ingress.nfs.happy
+
+# make sure mount can be reached over VIP, ensuring both that
+# keepalived is maintaining the VIP and that the nfs has bound to it
+- vip.exec:
+ host.a:
+ - mkdir /mnt/happy
+ - sleep 1
+ - mount -t nfs {{VIP0}}:/d1 /mnt/happy
+ - echo test > /mnt/happy/testfile
+ - sync
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml
new file mode 100644
index 000000000..3f4964978
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml
@@ -0,0 +1,89 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rgw foorgw --port 8800
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}}
+
+- vip.exec:
+ host.a:
+ - dnf install -y python3-boto3 || apt install -y python3-boto3
+ - /home/ubuntu/cephtest/cephadm shell radosgw-admin user create --uid foouser --display-name foo > /tmp/user.json
+
+- python:
+ host.a: |
+ import boto3
+ import json
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ bucket.create()
+ bucket.put_object(Key='myobject', Body='thebody')
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create rgw --bucket foobucket --cluster-id foo --pseudo-path /foobucket
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/foobucket /mnt/foo
+ - find /mnt/foo -ls
+ - grep thebody /mnt/foo/myobject
+ - echo test > /mnt/foo/newobject
+ - sync
+
+- python:
+ host.a: |
+ import boto3
+ import json
+ from io import BytesIO
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ data = BytesIO()
+ bucket.download_fileobj(Fileobj=data, Key='newobject')
+ print(data.getvalue())
+ assert data.getvalue().decode() == 'test\n'
+
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export rm foo /foobucket
+ - ceph nfs cluster rm foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml
new file mode 100644
index 000000000..721aecfc3
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml
@@ -0,0 +1,90 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rgw foorgw --port 8800
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}}
+
+- vip.exec:
+ host.a:
+ - dnf install -y python3-boto3 || apt install -y python3-boto3
+ - /home/ubuntu/cephtest/cephadm shell radosgw-admin user create --uid foouser --display-name foo > /tmp/user.json
+
+- python:
+ host.a: |
+ import boto3
+ import json
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ bucket.create()
+ bucket.put_object(Key='myobject', Body='thebody')
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create rgw --cluster-id foo --pseudo-path /foouser --user-id foouser
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/foouser /mnt/foo
+ - test -d /mnt/foo/foobucket
+ - find /mnt/foo -ls
+ - grep thebody /mnt/foo/foobucket/myobject
+ - echo test > /mnt/foo/foobucket/newobject
+ - sync
+
+- python:
+ host.a: |
+ import boto3
+ import json
+ from io import BytesIO
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ data = BytesIO()
+ bucket.download_fileobj(Fileobj=data, Key='newobject')
+ print(data.getvalue())
+ assert data.getvalue().decode() == 'test\n'
+
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export rm foo /foouser
+ - ceph nfs cluster rm foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml
new file mode 100644
index 000000000..b4e843df2
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml
@@ -0,0 +1,68 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+
+# deploy nfs + ingress
+- cephadm.apply:
+ specs:
+ - service_type: nfs
+ service_id: foo
+ placement:
+ count: 2
+ spec:
+ port: 12049
+ - service_type: ingress
+ service_id: nfs.foo
+ spec:
+ backend_service: nfs.foo
+ frontend_port: 2049
+ monitor_port: 9002
+ virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}"
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/fake /mnt/foo
+ - echo test > /mnt/foo/testfile
+ - sync
+
+# take each gateway down in turn and ensure things still work
+- cephadm.shell:
+ volumes:
+ - /mnt/foo:/mnt/foo
+ host.a:
+ - |
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ cat /mnt/foo/testfile
+ echo $haproxy > /mnt/foo/testfile
+ sync
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml
new file mode 100644
index 000000000..a47dd9d76
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml
@@ -0,0 +1,70 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}} --port 2999
+ - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/fake /mnt/foo -o port=2999
+ - echo test > /mnt/foo/testfile
+ - sync
+
+# take each gateway down in turn and ensure things still work
+- cephadm.shell:
+ volumes:
+ - /mnt/foo:/mnt/foo
+ host.a:
+ - |
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ cat /mnt/foo/testfile
+ echo $haproxy > /mnt/foo/testfile
+ sync
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
+
+# take each ganesha down in turn.
+# simulate "failure" by deleting the container
+- vip.exec:
+ all-hosts:
+ - |
+ echo "Check with $(hostname) ganesha(s) down..."
+ for c in `systemctl | grep ceph- | grep @nfs | awk '{print $1}'`; do
+ cid=`echo $c | sed 's/@/-/'`
+ id=`echo $c | cut -d @ -f 2 | sed 's/.service$//'`
+ fsid=`echo $c | cut -d @ -f 1 | cut -d - -f 2-`
+ echo "Removing daemon $id fsid $fsid..."
+ sudo $TESTDIR/cephadm rm-daemon --fsid $fsid --name $id
+
+ echo "Waking up cephadm..."
+ sudo $TESTDIR/cephadm shell -- ceph orch ps --refresh
+
+ while ! timeout 1 cat /mnt/foo/testfile ; do true ; done
+ echo "Mount is back!"
+ done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-keepalive-only.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-keepalive-only.yaml
new file mode 100644
index 000000000..ba5afed47
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-keepalive-only.yaml
@@ -0,0 +1,55 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+
+# deploy nfs + keepalive-only ingress service
+- cephadm.apply:
+ specs:
+ - service_type: nfs
+ service_id: foo
+ placement:
+ count: 1
+ spec:
+ port: 2049
+ virtual_ip: "{{VIP0}}"
+ - service_type: ingress
+ service_id: nfs.foo
+ placement:
+ count: 1
+ spec:
+ backend_service: nfs.foo
+ monitor_port: 9002
+ virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}"
+ keepalive_only: true
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+# export and mount
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+# make sure mount can be reached over VIP, ensuring both that
+# keepalived is maintaining the VIP and that the nfs has bound to it
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/fake /mnt/foo
+ - echo test > /mnt/foo/testfile
+ - sync
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml
new file mode 100644
index 000000000..194f4e9de
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml
@@ -0,0 +1,13 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.apply:
+ specs:
+ - service_type: nfs
+ service_id: foo
+- cephadm.wait_for_service:
+ service: nfs.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml
new file mode 100644
index 000000000..959c5aa77
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml
@@ -0,0 +1,12 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster create foo
+- cephadm.wait_for_service:
+ service: nfs.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nvmeof.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nvmeof.yaml
new file mode 100644
index 000000000..4c5e26740
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nvmeof.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply nvmeof foo
+- cephadm.wait_for_service:
+ service: nvmeof.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml
new file mode 100644
index 000000000..710edab73
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml
@@ -0,0 +1,60 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# deploy rgw + ingress
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+ - service_type: ingress
+ service_id: rgw.foo
+ placement:
+ count: 2
+ spec:
+ backend_service: rgw.foo
+ frontend_port: 9000
+ monitor_port: 9001
+ virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}"
+- cephadm.wait_for_service:
+ service: rgw.foo
+- cephadm.wait_for_service:
+ service: ingress.rgw.foo
+
+# take each component down in turn and ensure things still work
+- cephadm.shell:
+ host.a:
+ - |
+ echo "Check while healthy..."
+ curl http://{{VIP0}}:9000/
+
+ # stop each rgw in turn
+ echo "Check with each rgw stopped in turn..."
+ for rgw in `ceph orch ps | grep ^rgw.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $rgw
+ while ! ceph orch ps | grep $rgw | grep stopped; do sleep 1 ; done
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
+ ceph orch daemon start $rgw
+ while ! ceph orch ps | grep $rgw | grep running; do sleep 1 ; done
+ done
+
+ # stop each haproxy in turn
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.rgw.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
+
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml
new file mode 100644
index 000000000..cb2c6f4b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml
@@ -0,0 +1,12 @@
+tasks:
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count_per_host: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+- cephadm.wait_for_service:
+ service: rgw.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml b/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml
new file mode 100644
index 000000000..bb938848c
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml
@@ -0,0 +1,10 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - stat -c '%u %g' /var/log/ceph | grep '167 167'
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls | grep '^osd.all-available-devices '
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/% b/qa/suites/orch/cephadm/smoke-singlehost/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/%
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/.qa b/qa/suites/orch/cephadm/smoke-singlehost/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$ b/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/0-random-distro$
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml b/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml
new file mode 100644
index 000000000..ca6019c66
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml
@@ -0,0 +1,27 @@
+tasks:
+- cephadm:
+ roleless: true
+ single_host_defaults: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/2-services/.qa b/qa/suites/orch/cephadm/smoke-singlehost/2-services/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/2-services/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml b/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml b/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml
new file mode 100644
index 000000000..cb2c6f4b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml
@@ -0,0 +1,12 @@
+tasks:
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count_per_host: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+- cephadm.wait_for_service:
+ service: rgw.foo
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml b/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml
new file mode 100644
index 000000000..02f5b289c
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
diff --git a/qa/suites/orch/cephadm/smoke-small/% b/qa/suites/orch/cephadm/smoke-small/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/%
diff --git a/qa/suites/orch/cephadm/smoke-small/.qa b/qa/suites/orch/cephadm/smoke-small/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml
new file mode 120000
index 000000000..83fe02026
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/0-distro/centos_8.stream_container_tools_crun.yaml
@@ -0,0 +1 @@
+../.qa/distros/container-hosts/centos_8.stream_container_tools_crun.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/0-nvme-loop.yaml b/qa/suites/orch/cephadm/smoke-small/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/agent/.qa b/qa/suites/orch/cephadm/smoke-small/agent/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/agent/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/agent/off.yaml b/qa/suites/orch/cephadm/smoke-small/agent/off.yaml
new file mode 100644
index 000000000..f37c651bc
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/agent/off.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mgr:
+ mgr/cephadm/use_agent: false
diff --git a/qa/suites/orch/cephadm/smoke-small/agent/on.yaml b/qa/suites/orch/cephadm/smoke-small/agent/on.yaml
new file mode 100644
index 000000000..90ac298b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/agent/on.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mgr:
+ mgr/cephadm/use_agent: true
diff --git a/qa/suites/orch/cephadm/smoke-small/fixed-2.yaml b/qa/suites/orch/cephadm/smoke-small/fixed-2.yaml
new file mode 100644
index 000000000..61090a165
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/fixed-2.yaml
@@ -0,0 +1,29 @@
+roles:
+- - mon.a
+ - mgr.y
+ - osd.0
+ - client.0
+ - ceph.rgw.foo.a
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.1
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+- - mon.c
+ - mgr.z
+ - osd.2
+ - client.2
+ - node-exporter.c
+openstack:
+- volumes: # attached to each instance
+ count: 1
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke-small/mon_election b/qa/suites/orch/cephadm/smoke-small/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-small/start.yaml b/qa/suites/orch/cephadm/smoke-small/start.yaml
new file mode 100644
index 000000000..77f493ca1
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-small/start.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
+- cephadm.shell:
+ mon.a:
+ - stat -c '%u %g' /var/log/ceph | grep '167 167'
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls --format yaml
+ - ceph orch ls | grep '^osd '
diff --git a/qa/suites/orch/cephadm/smoke/% b/qa/suites/orch/cephadm/smoke/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/%
diff --git a/qa/suites/orch/cephadm/smoke/.qa b/qa/suites/orch/cephadm/smoke/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/0-distro b/qa/suites/orch/cephadm/smoke/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml b/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/agent/.qa b/qa/suites/orch/cephadm/smoke/agent/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/agent/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/agent/off.yaml b/qa/suites/orch/cephadm/smoke/agent/off.yaml
new file mode 100644
index 000000000..f37c651bc
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/agent/off.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mgr:
+ mgr/cephadm/use_agent: false
diff --git a/qa/suites/orch/cephadm/smoke/agent/on.yaml b/qa/suites/orch/cephadm/smoke/agent/on.yaml
new file mode 100644
index 000000000..90ac298b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/agent/on.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mgr:
+ mgr/cephadm/use_agent: true
diff --git a/qa/suites/orch/cephadm/smoke/fixed-2.yaml b/qa/suites/orch/cephadm/smoke/fixed-2.yaml
new file mode 100644
index 000000000..e93564aa9
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/fixed-2.yaml
@@ -0,0 +1,32 @@
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+ - ceph.rgw.foo.a
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+ - ceph.iscsi.iscsi.a
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke/mon_election b/qa/suites/orch/cephadm/smoke/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/start.yaml b/qa/suites/orch/cephadm/smoke/start.yaml
new file mode 100644
index 000000000..77f493ca1
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/start.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
+- cephadm.shell:
+ mon.a:
+ - stat -c '%u %g' /var/log/ceph | grep '167 167'
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls --format yaml
+ - ceph orch ls | grep '^osd '
diff --git a/qa/suites/orch/cephadm/thrash/% b/qa/suites/orch/cephadm/thrash/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/%
diff --git a/qa/suites/orch/cephadm/thrash/.qa b/qa/suites/orch/cephadm/thrash/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/0-distro b/qa/suites/orch/cephadm/thrash/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/1-start.yaml b/qa/suites/orch/cephadm/thrash/1-start.yaml
new file mode 100644
index 000000000..a1b89e44d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/1-start.yaml
@@ -0,0 +1,7 @@
+tasks:
+- install:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
diff --git a/qa/suites/orch/cephadm/thrash/2-thrash.yaml b/qa/suites/orch/cephadm/thrash/2-thrash.yaml
new file mode 100644
index 000000000..05e0f8e76
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/2-thrash.yaml
@@ -0,0 +1,26 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ conf:
+ osd:
+ osd debug reject backfill probability: .3
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd max backfills: 3
+ osd snap trim sleep: 2
+ osd delete sleep: 1
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 1
+ chance_pgnum_shrink: 1
+ chance_pgpnum_fix: 1
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/.qa b/qa/suites/orch/cephadm/thrash/3-tasks/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml
new file mode 120000
index 000000000..34e657e04
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/rados_api_tests.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml
new file mode 120000
index 000000000..dad17e0de
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/radosbench.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml
new file mode 120000
index 000000000..6aa66aa37
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/small-objects.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml
new file mode 120000
index 000000000..c9cc4cd3e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/snaps-few-objects.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/fixed-2.yaml b/qa/suites/orch/cephadm/thrash/fixed-2.yaml
new file mode 120000
index 000000000..5c3e0593c
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/fixed-2.yaml
@@ -0,0 +1 @@
+../smoke/fixed-2.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/msgr b/qa/suites/orch/cephadm/thrash/msgr
new file mode 120000
index 000000000..57bee80db
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/msgr
@@ -0,0 +1 @@
+.qa/msgr \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/root.yaml b/qa/suites/orch/cephadm/thrash/root.yaml
new file mode 100644
index 000000000..bedb31d5d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/root.yaml
@@ -0,0 +1,3 @@
+overrides:
+ cephadm:
+ cephadm_mode: root
diff --git a/qa/suites/orch/cephadm/upgrade/% b/qa/suites/orch/cephadm/upgrade/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/%
diff --git a/qa/suites/orch/cephadm/upgrade/.qa b/qa/suites/orch/cephadm/upgrade/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/.qa b/qa/suites/orch/cephadm/upgrade/1-start-distro/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
new file mode 100644
index 000000000..bb9a220e6
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
@@ -0,0 +1,39 @@
+os_type: centos
+os_version: "8.stream"
+
+tasks:
+- pexec:
+ all:
+ - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
+ - sudo dnf -y module reset container-tools
+ - sudo dnf -y module install container-tools
+ - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
+- cephadm:
+ image: quay.io/ceph/ceph:v16.2.0
+ cephadm_branch: v16.2.0
+ cephadm_git_url: https://github.com/ceph/ceph
+ # avoid --cap-add=PTRACE + --privileged for older cephadm versions
+ allow_ptrace: false
+ avoid_pacific_features: true
+
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml
new file mode 100644
index 000000000..d3d9de83e
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml
@@ -0,0 +1,33 @@
+os_type: ubuntu
+os_version: "20.04"
+
+tasks:
+- cephadm:
+ image: quay.io/ceph/ceph:v16.2.0
+ cephadm_branch: v16.2.0
+ cephadm_git_url: https://github.com/ceph/ceph
+ # avoid --cap-add=PTRACE + --privileged for older cephadm versions
+ allow_ptrace: false
+ avoid_pacific_features: true
+
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
diff --git a/qa/suites/orch/cephadm/upgrade/2-repo_digest/.qa b/qa/suites/orch/cephadm/upgrade/2-repo_digest/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/2-repo_digest/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml b/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml
diff --git a/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml b/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml
new file mode 100644
index 000000000..2e6bbfd92
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml
@@ -0,0 +1,4 @@
+tasks:
+- cephadm.shell:
+ mon.a:
+ - ceph config set mgr mgr/cephadm/use_repo_digest false --force
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa b/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml
new file mode 100644
index 000000000..f10a49bea
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml
@@ -0,0 +1,21 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ # setup rgw
+ - radosgw-admin realm create --rgw-realm=r --default
+ - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
+ - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
+ - radosgw-admin period update --rgw-realm=r --commit
+ - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000
+ # simple rgw spec (will have no "spec" field) to make sure that works with rgw spec migration
+ - ceph orch apply rgw smpl
+ # setup iscsi
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply iscsi foo u p
+ - sleep 120
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph config set global log_to_journald false --force
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml
new file mode 100644
index 000000000..280714e4e
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml
@@ -0,0 +1,132 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ # setup rgw
+ - radosgw-admin realm create --rgw-realm=r --default
+ - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
+ - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
+ - radosgw-admin period update --rgw-realm=r --commit
+ - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000
+ # setup iscsi
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply iscsi foo u p
+ - sleep 180
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph config set global log_to_journald false --force
+ # get some good info on the state of things pre-upgrade. Useful for debugging
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ - ceph orch ls
+ # doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code
+ # until there is a stable version that contains it, we can test by manually upgrading a mgr daemon
+ - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
+ - ceph orch ps --refresh
+ - sleep 180
+ # gather more possible debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ - ceph health detail
+ # check that there are two different versions found for mgr daemon (which implies we upgraded one)
+ - ceph versions | jq -e '.mgr | length == 2'
+ - ceph mgr fail
+ - sleep 180
+ # now try upgrading the other mgr
+ - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
+ - ceph orch ps --refresh
+ - sleep 180
+ # gather more possible debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph health detail
+ - ceph -s
+ - ceph mgr fail
+ - sleep 180
+ # gather more debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ - ceph health detail
+ # now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs
+ - ceph versions | jq -e '.mgr | length == 1'
+ - ceph mgr fail
+ - sleep 180
+ # debugging info
+ - ceph orch ps
+ - ceph orch ls
+ - ceph versions
+ # to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on a new version
+ # also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ # verify only one version found for mgrs and that their version hash matches what we are upgrading to
+ - ceph versions | jq -e '.mgr | length == 1'
+ - ceph versions | jq -e '.mgr | keys' | grep $sha1
+ # verify overall we still see two versions, basically to make sure --daemon-types wasn't ignored and all daemons upgraded
+ - ceph versions | jq -e '.overall | length == 2'
+ # check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
+ - ceph orch upgrade status
+ - ceph health detail
+ # upgrade only the mons on one of the two hosts
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify two different version seen for mons
+ - ceph versions | jq -e '.mon | length == 2'
+ - ceph orch upgrade status
+ - ceph health detail
+ # upgrade mons on the other hosts
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all mons now on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.mon | length == 1'
+ - ceph versions | jq -e '.mon | keys' | grep $sha1
+ # verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
+ - ceph orch upgrade status
+ - ceph health detail
+ # upgrade exactly 2 osd daemons
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify two different versions now seen for osds
+ - ceph versions | jq -e '.osd | length == 2'
+ # verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
+ - ceph orch upgrade status
+ - ceph health detail
+ # upgrade one more osd
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions | jq -e '.osd | length == 2'
+ # verify now 8 daemons have been upgraded
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
+ # upgrade the rest of the osds
+ - ceph orch upgrade status
+ - ceph health detail
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all osds are now on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.osd | length == 1'
+ - ceph versions | jq -e '.osd | keys' | grep $sha1
+ - ceph orch upgrade status
+ - ceph health detail
+ # upgrade the rgw daemons using --services
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all rgw daemons on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.rgw | length == 1'
+ - ceph versions | jq -e '.rgw | keys' | grep $sha1
+ - ceph orch upgrade status
+ - ceph health detail
+ # run upgrade one more time with no filter parameters to make sure anything left gets upgraded
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
diff --git a/qa/suites/orch/cephadm/upgrade/4-wait.yaml b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
new file mode 100644
index 000000000..4010c58ed
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions
+ - echo "wait for servicemap items w/ changing names to refresh"
+ - sleep 60
+ - ceph orch ps
+ - ceph versions
+ - ceph orch upgrade status
+ - ceph health detail
+ - ceph versions | jq -e '.overall | length == 1'
+ - ceph versions | jq -e '.overall | keys' | grep $sha1
+ - ceph orch ls | grep '^osd '
diff --git a/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml b/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml
new file mode 100644
index 000000000..799458bc5
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml
@@ -0,0 +1,6 @@
+tasks:
+- cephadm.shell:
+ mon.a:
+ - ceph orch upgrade ls
+ - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0
+ - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2
diff --git a/qa/suites/orch/cephadm/upgrade/agent b/qa/suites/orch/cephadm/upgrade/agent
new file mode 120000
index 000000000..154924209
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/agent
@@ -0,0 +1 @@
+../smoke/agent \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/mon_election b/qa/suites/orch/cephadm/upgrade/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/% b/qa/suites/orch/cephadm/with-work/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/%
diff --git a/qa/suites/orch/cephadm/with-work/.qa b/qa/suites/orch/cephadm/with-work/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/0-distro b/qa/suites/orch/cephadm/with-work/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/fixed-2.yaml b/qa/suites/orch/cephadm/with-work/fixed-2.yaml
new file mode 120000
index 000000000..5c3e0593c
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/fixed-2.yaml
@@ -0,0 +1 @@
+../smoke/fixed-2.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/mode/.qa b/qa/suites/orch/cephadm/with-work/mode/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/mode/packaged.yaml b/qa/suites/orch/cephadm/with-work/mode/packaged.yaml
new file mode 100644
index 000000000..ba8d43218
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/packaged.yaml
@@ -0,0 +1,5 @@
+overrides:
+ cephadm:
+ cephadm_mode: cephadm-package
+ install:
+ extra_packages: [cephadm]
diff --git a/qa/suites/orch/cephadm/with-work/mode/root.yaml b/qa/suites/orch/cephadm/with-work/mode/root.yaml
new file mode 100644
index 000000000..bedb31d5d
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/root.yaml
@@ -0,0 +1,3 @@
+overrides:
+ cephadm:
+ cephadm_mode: root
diff --git a/qa/suites/orch/cephadm/with-work/mon_election b/qa/suites/orch/cephadm/with-work/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/msgr b/qa/suites/orch/cephadm/with-work/msgr
new file mode 120000
index 000000000..57bee80db
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/msgr
@@ -0,0 +1 @@
+.qa/msgr \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/start.yaml b/qa/suites/orch/cephadm/with-work/start.yaml
new file mode 100644
index 000000000..a1b89e44d
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/start.yaml
@@ -0,0 +1,7 @@
+tasks:
+- install:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
diff --git a/qa/suites/orch/cephadm/with-work/tasks/.qa b/qa/suites/orch/cephadm/with-work/tasks/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml b/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml
new file mode 120000
index 000000000..2ce80f969
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/basic/tasks/rados_api_tests.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml b/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml
new file mode 120000
index 000000000..210ad8f18
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/basic/tasks/rados_python.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/tasks/rotate-keys.yaml b/qa/suites/orch/cephadm/with-work/tasks/rotate-keys.yaml
new file mode 100644
index 000000000..5b91c6ed3
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/rotate-keys.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm.shell:
+ mon.a:
+ - |
+ set -ex
+ for f in osd.0 osd.1 osd.2 osd.3 osd.4 osd.5 osd.6 osd.7 mgr.y mgr.x
+ do
+ echo "rotating key for $f"
+ K=$(ceph auth get-key $f)
+ NK="$K"
+ ceph orch daemon rotate-key $f
+ while [ "$K" == "$NK" ]; do
+ sleep 5
+ NK=$(ceph auth get-key $f)
+ done
+ done
diff --git a/qa/suites/orch/cephadm/workunits/% b/qa/suites/orch/cephadm/workunits/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/%
diff --git a/qa/suites/orch/cephadm/workunits/.qa b/qa/suites/orch/cephadm/workunits/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/0-distro b/qa/suites/orch/cephadm/workunits/0-distro
new file mode 120000
index 000000000..4b341719d
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/0-distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/agent b/qa/suites/orch/cephadm/workunits/agent
new file mode 120000
index 000000000..154924209
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/agent
@@ -0,0 +1 @@
+../smoke/agent \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/mon_election b/qa/suites/orch/cephadm/workunits/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/.qa b/qa/suites/orch/cephadm/workunits/task/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml b/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml
new file mode 100644
index 000000000..e04fc1eea
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml
@@ -0,0 +1,11 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- install:
+- exec:
+ mon.a:
+ - yum install -y python3 || apt install -y python3
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_adoption.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml b/qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml
new file mode 100644
index 000000000..7bf51f719
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_ca_signed_key.yaml
@@ -0,0 +1,31 @@
+roles:
+- - host.a
+ - mon.a
+ - mgr.a
+ - osd.0
+ - client.0
+- - host.b
+ - mon.b
+ - mgr.b
+ - osd.1
+ - client.1
+overrides:
+ cephadm:
+ use-ca-signed-key: True
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - |
+ set -ex
+ HOSTNAMES=$(ceph orch host ls --format json | jq -r '.[] | .hostname')
+ for host in $HOSTNAMES; do
+ # do a check-host on each host to make sure it's reachable
+ ceph cephadm check-host ${host} 2> ${host}-ok.txt
+ HOST_OK=$(cat ${host}-ok.txt)
+ if ! grep -q "Host looks OK" <<< "$HOST_OK"; then
+ printf "Failed host check:\n\n$HOST_OK"
+ exit 1
+ fi
+ done
diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml b/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml
new file mode 100644
index 000000000..4d253517c
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml
@@ -0,0 +1,11 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- install:
+- exec:
+ mon.a:
+ - yum install -y python3 || apt install -y python3
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_cephadm.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml b/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml
new file mode 100644
index 000000000..4a1ac88de
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml
@@ -0,0 +1,8 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- workunit:
+ no_coverage_and_limits: true
+ clients:
+ client.0:
+ - cephadm/test_repos.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/+ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/+
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/+
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/.qa b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml
new file mode 120000
index 000000000..7a86f967f
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/centos_8.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.stream_container_tools.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml
new file mode 100644
index 000000000..19d302c87
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_iscsi_container/test_iscsi_container.yaml
@@ -0,0 +1,21 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply iscsi foo u p
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_iscsi_pids_limit.sh
+ - cephadm/test_iscsi_etc_hosts.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml b/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml
new file mode 100644
index 000000000..ec65fb116
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml
@@ -0,0 +1,17 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply mds a
+- cephfs_test_runner:
+ modules:
+ - tasks.cephadm_cases.test_cli
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
new file mode 100644
index 000000000..2a33dc839
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
@@ -0,0 +1,45 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+- - host.b
+ - osd.3
+ - osd.4
+ - osd.5
+ - mon.b
+ - mgr.b
+ - client.1
+- - host.c
+ - osd.6
+ - osd.7
+ - osd.8
+ - mon.c
+ - mgr.c
+ - client.2
+- - host.d
+ - osd.9
+ - osd.10
+ - osd.11
+ - mon.d
+ - mgr.d
+ - client.3
+- - host.e
+ - osd.12
+ - osd.13
+ - osd.14
+ - mon.e
+ - mgr.e
+ - client.4
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply mds a
+- cephfs_test_runner:
+ modules:
+ - tasks.cephadm_cases.test_cli_mon
diff --git a/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml b/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
new file mode 100644
index 000000000..976e3730c
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_rgw_multisite.yaml
@@ -0,0 +1,40 @@
+roles:
+- - host.a
+ - mon.a
+ - mgr.a
+ - osd.0
+- - host.b
+ - mon.b
+ - mgr.b
+ - osd.1
+- - host.c
+ - mon.c
+ - osd.2
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph mgr module enable rgw
+- rgw_module.apply:
+ specs:
+ - rgw_realm: myrealm1
+ rgw_zonegroup: myzonegroup1
+ rgw_zone: myzone1
+ spec:
+ rgw_frontend_port: 5500
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ while true; do TOKEN=$(ceph rgw realm tokens | jq -r '.[0].token'); echo $TOKEN; if [ "$TOKEN" != "master zone has no endpoint" ]; then break; fi; sleep 5; done
+ TOKENS=$(ceph rgw realm tokens)
+ echo $TOKENS | jq --exit-status '.[0].realm == "myrealm1"'
+ echo $TOKENS | jq --exit-status '.[0].token'
+ TOKEN_JSON=$(ceph rgw realm tokens | jq -r '.[0].token' | base64 --decode)
+ echo $TOKEN_JSON | jq --exit-status '.realm_name == "myrealm1"'
+ echo $TOKEN_JSON | jq --exit-status '.endpoint | test("http://.+:\\d+")'
+ echo $TOKEN_JSON | jq --exit-status '.realm_id | test("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")'
+ echo $TOKEN_JSON | jq --exit-status '.access_key'
+ echo $TOKEN_JSON | jq --exit-status '.secret'
diff --git a/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml b/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
new file mode 100644
index 000000000..6d9bd1525
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_set_mon_crush_locations.yaml
@@ -0,0 +1,62 @@
+roles:
+- - host.a
+ - osd.0
+ - mon.a
+ - mgr.a
+- - host.b
+ - osd.1
+ - mon.b
+ - mgr.b
+- - host.c
+ - osd.2
+ - mon.c
+tasks:
+- install:
+- cephadm:
+- cephadm.apply:
+ specs:
+ - service_type: mon
+ service_id: foo
+ placement:
+ count: 3
+ spec:
+ crush_locations:
+ host.a:
+ - datacenter=a
+ host.b:
+ - datacenter=b
+ - rack=2
+ host.c:
+ - datacenter=a
+ - rack=3
+- cephadm.shell:
+ host.a:
+ - |
+ set -ex
+ # since we don't know the real hostnames before the test, the next
+ # bit is in order to replace the fake hostnames "host.a/b/c" with
+ # the actual names cephadm knows the host by within the mon spec
+ ceph orch host ls --format json | jq -r '.[] | .hostname' > realnames
+ echo $'host.a\nhost.b\nhost.c' > fakenames
+ echo $'a\nb\nc' > mon_ids
+ echo $'{datacenter=a}\n{datacenter=b,rack=2}\n{datacenter=a,rack=3}' > crush_locs
+ ceph orch ls --service-name mon --export > mon.yaml
+ MONSPEC=`cat mon.yaml`
+ echo "$MONSPEC"
+ while read realname <&3 && read fakename <&4; do
+ MONSPEC="${MONSPEC//$fakename/$realname}"
+ done 3<realnames 4<fakenames
+ echo "$MONSPEC" > mon.yaml
+ cat mon.yaml
+ # now the spec should have the real hostnames, so let's re-apply
+ ceph orch apply -i mon.yaml
+ sleep 90
+ ceph orch ps --refresh
+ ceph orch ls --service-name mon --export > mon.yaml; ceph orch apply -i mon.yaml
+ sleep 90
+ ceph mon dump
+ ceph mon dump --format json
+ # verify all the crush locations got set from "ceph mon dump" output
+ while read monid <&3 && read crushloc <&4; do
+ ceph mon dump --format json | jq --arg monid "$monid" --arg crushloc "$crushloc" -e '.mons | .[] | select(.name == $monid) | .crush_location == $crushloc'
+ done 3<mon_ids 4<crush_locs