summaryrefslogtreecommitdiffstats
path: root/qa/suites/orch/cephadm
diff options
context:
space:
mode:
Diffstat (limited to 'qa/suites/orch/cephadm')
l---------qa/suites/orch/cephadm/.qa1
-rw-r--r--qa/suites/orch/cephadm/dashboard/%0
l---------qa/suites/orch/cephadm/dashboard/.qa1
l---------qa/suites/orch/cephadm/dashboard/0-distro/.qa1
l---------qa/suites/orch/cephadm/dashboard/0-distro/centos_8.stream_container_tools.yaml1
l---------qa/suites/orch/cephadm/dashboard/0-distro/ignorelist_health.yaml1
-rw-r--r--qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml23
l---------qa/suites/orch/cephadm/mds_upgrade_sequence1
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/%0
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa1
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/.qa1
l---------qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/centos_8.stream_container_tools.yaml1
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml8
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml6
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml8
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml29
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml29
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml41
-rw-r--r--qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml10
-rw-r--r--qa/suites/orch/cephadm/orchestrator_cli/%0
l---------qa/suites/orch/cephadm/orchestrator_cli/.qa1
l---------qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$1
l---------qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml1
-rw-r--r--qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml18
-rw-r--r--qa/suites/orch/cephadm/osds/%0
l---------qa/suites/orch/cephadm/osds/.qa1
l---------qa/suites/orch/cephadm/osds/0-distro1
l---------qa/suites/orch/cephadm/osds/0-nvme-loop.yaml1
-rw-r--r--qa/suites/orch/cephadm/osds/1-start.yaml25
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml13
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml17
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml15
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml16
-rw-r--r--qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml20
l---------qa/suites/orch/cephadm/rbd_iscsi1
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/%0
l---------qa/suites/orch/cephadm/smoke-roleless/.qa1
l---------qa/suites/orch/cephadm/smoke-roleless/0-distro1
l---------qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml1
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/1-start.yaml24
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml0
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml40
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml8
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml9
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml89
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml90
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml68
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml70
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml13
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml60
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-roleless/3-final.yaml10
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/%0
l---------qa/suites/orch/cephadm/smoke-singlehost/0-distro$1
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml27
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml0
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml12
-rw-r--r--qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml8
-rw-r--r--qa/suites/orch/cephadm/smoke/%0
l---------qa/suites/orch/cephadm/smoke/.qa1
l---------qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml1
l---------qa/suites/orch/cephadm/smoke/distro1
-rw-r--r--qa/suites/orch/cephadm/smoke/fixed-2.yaml32
l---------qa/suites/orch/cephadm/smoke/mon_election1
-rw-r--r--qa/suites/orch/cephadm/smoke/start.yaml16
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/%0
l---------qa/suites/orch/cephadm/thrash-old-clients/.qa1
l---------qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa1
l---------qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml1
l---------qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml1
l---------qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml32
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml31
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml32
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml32
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml25
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml25
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml20
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml34
l---------qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml0
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml5
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml6
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml7
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/clusters/+0
l---------qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml4
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml13
l---------qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml6
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml0
l---------qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa1
l---------qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml1
l---------qa/suites/orch/cephadm/thrash-old-clients/mon_election1
l---------qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml9
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml10
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml12
l---------qa/suites/orch/cephadm/thrash-old-clients/rados.yaml1
l---------qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml25
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml24
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml26
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml22
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml0
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml24
l---------qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml1
l---------qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa1
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml34
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml33
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml7
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml13
-rw-r--r--qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml8
-rw-r--r--qa/suites/orch/cephadm/thrash/%0
l---------qa/suites/orch/cephadm/thrash/.qa1
l---------qa/suites/orch/cephadm/thrash/0-distro1
-rw-r--r--qa/suites/orch/cephadm/thrash/1-start.yaml7
-rw-r--r--qa/suites/orch/cephadm/thrash/2-thrash.yaml26
l---------qa/suites/orch/cephadm/thrash/3-tasks/.qa1
l---------qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml1
l---------qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml1
l---------qa/suites/orch/cephadm/thrash/fixed-2.yaml1
l---------qa/suites/orch/cephadm/thrash/msgr1
-rw-r--r--qa/suites/orch/cephadm/thrash/root.yaml3
-rw-r--r--qa/suites/orch/cephadm/upgrade/%0
l---------qa/suites/orch/cephadm/upgrade/.qa1
-rw-r--r--qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml43
-rw-r--r--qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml37
-rw-r--r--qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml37
-rw-r--r--qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml0
-rw-r--r--qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml4
l---------qa/suites/orch/cephadm/upgrade/3-upgrade/.qa1
-rw-r--r--qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml13
-rw-r--r--qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml111
-rw-r--r--qa/suites/orch/cephadm/upgrade/4-wait.yaml14
-rw-r--r--qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml6
l---------qa/suites/orch/cephadm/upgrade/mon_election1
-rw-r--r--qa/suites/orch/cephadm/with-work/%0
l---------qa/suites/orch/cephadm/with-work/.qa1
l---------qa/suites/orch/cephadm/with-work/0-distro1
l---------qa/suites/orch/cephadm/with-work/fixed-2.yaml1
l---------qa/suites/orch/cephadm/with-work/mode/.qa1
-rw-r--r--qa/suites/orch/cephadm/with-work/mode/packaged.yaml5
-rw-r--r--qa/suites/orch/cephadm/with-work/mode/root.yaml3
l---------qa/suites/orch/cephadm/with-work/mon_election1
l---------qa/suites/orch/cephadm/with-work/msgr1
-rw-r--r--qa/suites/orch/cephadm/with-work/start.yaml7
l---------qa/suites/orch/cephadm/with-work/tasks/.qa1
l---------qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml1
l---------qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml1
-rw-r--r--qa/suites/orch/cephadm/workunits/%0
l---------qa/suites/orch/cephadm/workunits/.qa1
l---------qa/suites/orch/cephadm/workunits/0-distro/.qa1
l---------qa/suites/orch/cephadm/workunits/0-distro/centos_8.stream_container_tools.yaml1
l---------qa/suites/orch/cephadm/workunits/mon_election1
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_adoption.yaml11
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml11
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml8
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_nfs.yaml17
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml17
-rw-r--r--qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml45
164 files changed, 1887 insertions, 0 deletions
diff --git a/qa/suites/orch/cephadm/.qa b/qa/suites/orch/cephadm/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/dashboard/% b/qa/suites/orch/cephadm/dashboard/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/%
diff --git a/qa/suites/orch/cephadm/dashboard/.qa b/qa/suites/orch/cephadm/dashboard/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/dashboard/0-distro/.qa b/qa/suites/orch/cephadm/dashboard/0-distro/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/0-distro/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.stream_container_tools.yaml
new file mode 120000
index 000000000..7a86f967f
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/0-distro/centos_8.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.stream_container_tools.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/dashboard/0-distro/ignorelist_health.yaml b/qa/suites/orch/cephadm/dashboard/0-distro/ignorelist_health.yaml
new file mode 120000
index 000000000..5cb891a95
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/0-distro/ignorelist_health.yaml
@@ -0,0 +1 @@
+.qa/cephfs/overrides/ignorelist_health.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml b/qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml
new file mode 100644
index 000000000..cb6ffb22f
--- /dev/null
+++ b/qa/suites/orch/cephadm/dashboard/task/test_e2e.yaml
@@ -0,0 +1,23 @@
+roles:
+# 3 osd roles on host.a is required for cephadm task. It checks if the cluster is healthy.
+# More daemons will be deployed on both hosts in e2e tests.
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+- - host.b
+ - client.1
+tasks:
+- install:
+- cephadm:
+- workunit:
+ clients:
+ client.1:
+ - cephadm/create_iscsi_disks.sh
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_dashboard_e2e.sh
diff --git a/qa/suites/orch/cephadm/mds_upgrade_sequence b/qa/suites/orch/cephadm/mds_upgrade_sequence
new file mode 120000
index 000000000..24aa41c10
--- /dev/null
+++ b/qa/suites/orch/cephadm/mds_upgrade_sequence
@@ -0,0 +1 @@
+.qa/suites/fs/upgrade/mds_upgrade_sequence/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/% b/qa/suites/orch/cephadm/mgr-nfs-upgrade/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/%
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/.qa b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/centos_8.stream_container_tools.yaml
new file mode 120000
index 000000000..7a86f967f
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/0-distro/centos_8.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.stream_container_tools.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
new file mode 100644
index 000000000..991562b94
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.4.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: docker.io/ceph/ceph:v16.2.4
+ cephadm_branch: v16.2.4
+ cephadm_git_url: https://github.com/ceph/ceph
+ # needed for v16.2.4 due to --skip-admin-label
+ avoid_pacific_features: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
new file mode 100644
index 000000000..2568f23fa
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/16.2.5.yaml
@@ -0,0 +1,6 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: docker.io/ceph/ceph:v16.2.5
+ cephadm_branch: v16.2.5
+ cephadm_git_url: https://github.com/ceph/ceph
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml
new file mode 100644
index 000000000..16670e7ec
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-bootstrap/octopus.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm:
+ roleless: true
+ image: docker.io/ceph/ceph:v15
+ cephadm_branch: octopus
+ cephadm_git_url: https://github.com/ceph/ceph
+ avoid_pacific_features: true
+ add_mons_via_daemon_add: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
new file mode 100644
index 000000000..2d9f09a4e
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/1-start.yaml
@@ -0,0 +1,29 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+- - host.b
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
new file mode 100644
index 000000000..34680fc8a
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml
@@ -0,0 +1,29 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+
+- cephadm.wait_for_service:
+ service: mds.foofs
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2
+ - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+ # we can't do wait_for_service here because with octopus it's nfs.ganesha-foo not nfs.foo
+ - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done
+ - echo test > /mnt/foo/testfile
+ - sync
+
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
new file mode 100644
index 000000000..362e03734
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/3-upgrade-with-workload.yaml
@@ -0,0 +1,41 @@
+tasks:
+- parallel:
+ - upgrade-tasks
+ - workload-tasks
+
+upgrade-tasks:
+ sequential:
+ - cephadm.shell:
+ env: [sha1]
+ host.a:
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph config set global log_to_journald false --force
+ - ceph mgr module enable nfs --force
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
+ - cephadm.shell:
+ env: [sha1]
+ host.a:
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions
+ - echo "wait for servicemap items w/ changing names to refresh"
+ - sleep 60
+ - ceph orch ps
+ - ceph versions
+ - ceph versions | jq -e '.overall | length == 1'
+ - ceph versions | jq -e '.overall | keys' | grep $sha1
+
+ # this should be a no-op, but confirms nfs.ganesha-foo was remapped to nfs.foo
+ - cephadm.wait_for_service:
+ service: nfs.foo
+
+workload-tasks:
+ sequential:
+ - exec:
+ host.a:
+ - cd /mnt/foo && dbench 5 -t 600 || true # might fail with ESTALE
+ # make sure mount works
+ - umount /mnt/foo
+ - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done
+ - cd /mnt/foo && dbench 5 -t 5
diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
new file mode 100644
index 000000000..2a834ead4
--- /dev/null
+++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/4-final.yaml
@@ -0,0 +1,10 @@
+tasks:
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster ls | grep foo
+ - ceph nfs export ls foo --detailed
+ - rados -p .nfs --all ls -
+ - ceph config get mgr mgr/cephadm/migration_current | grep 5
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/% b/qa/suites/orch/cephadm/orchestrator_cli/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/%
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/.qa b/qa/suites/orch/cephadm/orchestrator_cli/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
new file mode 120000
index 000000000..d2dffb181
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/0-random-distro$
@@ -0,0 +1 @@
+../smoke/distro \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml b/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml
new file mode 120000
index 000000000..8a0b9123b
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/2-node-mgr.yaml
@@ -0,0 +1 @@
+.qa/clusters/2-node-mgr.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
new file mode 100644
index 000000000..564a2eb02
--- /dev/null
+++ b/qa/suites/orch/cephadm/orchestrator_cli/orchestrator_cli.yaml
@@ -0,0 +1,18 @@
+
+tasks:
+ - install:
+ - ceph:
+ # tests may leave mgrs broken, so don't try and call into them
+ # to invoke e.g. pg dump during teardown.
+ wait-for-scrub: false
+ log-ignorelist:
+ - overall HEALTH_
+ - \(MGR_DOWN\)
+ - \(DEVICE_IDENT_ON\)
+ - \(DEVICE_FAULT_ON\)
+ - \(PG_
+ - replacing it with standby
+ - No standby daemons available
+ - cephfs_test_runner:
+ modules:
+ - tasks.mgr.test_orchestrator_cli \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/% b/qa/suites/orch/cephadm/osds/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/%
diff --git a/qa/suites/orch/cephadm/osds/.qa b/qa/suites/orch/cephadm/osds/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/0-distro b/qa/suites/orch/cephadm/osds/0-distro
new file mode 120000
index 000000000..d2dffb181
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/0-distro
@@ -0,0 +1 @@
+../smoke/distro \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml b/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/osds/1-start.yaml b/qa/suites/orch/cephadm/osds/1-start.yaml
new file mode 100644
index 000000000..4331d7c66
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/1-start.yaml
@@ -0,0 +1,25 @@
+tasks:
+- cephadm:
+ roleless: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls | grep '^osd.all-available-devices '
+roles:
+- - host.a
+ - client.0
+- - host.b
+ - client.1
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml
new file mode 100644
index 000000000..16413aba8
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml
@@ -0,0 +1,13 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ ceph osd tree
+ for osd in `ceph osd ls` ; do
+ ceph orch osd rm $osd --force --zap --replace
+ done
+ while ceph orch osd rm ls | wc | grep ^1 ; do sleep 10 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml
new file mode 100644
index 000000000..09be72f11
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-add.yaml
@@ -0,0 +1,17 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep $DEVID | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep $DEVID | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ ceph orch device zap $HOST $DEV --force
+ ceph orch daemon add osd $HOST:$DEV
+ while ! ceph osd dump | grep osd.1 | grep up ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml
new file mode 100644
index 000000000..8f07f6d53
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml
@@ -0,0 +1,15 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep "$DEVID" | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep "$DEVID" | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm --zap --replace 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ while ! ceph osd dump | grep osd.1 | grep "up\s*in" ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml
new file mode 100644
index 000000000..78161aa49
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-wait.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ ceph orch device ls
+ DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}')
+ HOST=$(ceph orch device ls | grep $DEVID | awk '{print $1}')
+ DEV=$(ceph orch device ls | grep $DEVID | awk '{print $2}')
+ echo "host $HOST, dev $DEV, devid $DEVID"
+ ceph orch osd rm 1
+ while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done
+ ceph orch device zap $HOST $DEV --force
+ while ! ceph osd dump | grep osd.1 | grep up ; do sleep 5 ; done
diff --git a/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml b/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml
new file mode 100644
index 000000000..a971a02e4
--- /dev/null
+++ b/qa/suites/orch/cephadm/osds/2-ops/rmdir-reactivate.yaml
@@ -0,0 +1,20 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - |
+ set -e
+ set -x
+ ceph orch ps
+ HOST=$(hostname -s)
+ OSD=$(ceph orch ps $HOST | grep osd | head -n 1 | awk '{print $1}')
+ echo "host $HOST, osd $OSD"
+ ceph orch daemon stop $OSD
+ while ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done
+ ceph auth export $OSD > k
+ ceph orch daemon rm $OSD --force
+ ceph orch ps --refresh
+ while ceph orch ps | grep $OSD ; do sleep 5 ; done
+ ceph auth add $OSD -i k
+ ceph cephadm osd activate $HOST
+ while ! ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done
+- cephadm.healthy:
diff --git a/qa/suites/orch/cephadm/rbd_iscsi b/qa/suites/orch/cephadm/rbd_iscsi
new file mode 120000
index 000000000..f0073a119
--- /dev/null
+++ b/qa/suites/orch/cephadm/rbd_iscsi
@@ -0,0 +1 @@
+.qa/suites/rbd/iscsi \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/% b/qa/suites/orch/cephadm/smoke-roleless/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/%
diff --git a/qa/suites/orch/cephadm/smoke-roleless/.qa b/qa/suites/orch/cephadm/smoke-roleless/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/0-distro b/qa/suites/orch/cephadm/smoke-roleless/0-distro
new file mode 120000
index 000000000..d2dffb181
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/0-distro
@@ -0,0 +1 @@
+../smoke/distro \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml b/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml b/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml
new file mode 100644
index 000000000..018356f8f
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/1-start.yaml
@@ -0,0 +1,24 @@
+tasks:
+- cephadm:
+ roleless: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - client.0
+- - host.b
+ - client.1
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/basic.yaml
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml
new file mode 100644
index 000000000..f00800471
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/client-keyring.yaml
@@ -0,0 +1,40 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch host label add `hostname` foo
+ - ceph auth get-or-create client.foo mon 'allow r'
+ - ceph orch client-keyring set client.foo label:foo --mode 770 --owner 11111:22222
+- exec:
+ host.a:
+ - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx---
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222
+ - test -e /etc/ceph/ceph.conf
+- exec:
+ host.b:
+ - test ! -e /etc/ceph/ceph.client.foo.keyring
+- cephadm.shell:
+ host.b:
+ - ceph orch host label add `hostname` foo
+- exec:
+ host.b:
+ - while ! test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep rwxrwx---
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 11111
+ - ls -al /etc/ceph/ceph.client.foo.keyring | grep 22222
+- cephadm.shell:
+ host.b:
+ - ceph orch host label rm `hostname` foo
+- exec:
+ host.b:
+ - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
+- exec:
+ host.a:
+ - test -e /etc/ceph/ceph.client.foo.keyring
+- cephadm.shell:
+ host.a:
+ - ceph orch client-keyring rm client.foo
+- exec:
+ host.a:
+ - while test -e /etc/ceph/ceph.client.foo.keyring ; do sleep 1 ; done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml
new file mode 100644
index 000000000..7f57076db
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/iscsi.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph osd pool create foo
+ - rbd pool init foo
+ - ceph orch apply iscsi foo u p
+- cephadm.wait_for_service:
+ service: iscsi.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml
new file mode 100644
index 000000000..681e1e04a
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/mirror.yaml
@@ -0,0 +1,9 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rbd-mirror "--placement=*"
+ - ceph orch apply cephfs-mirror "--placement=*"
+- cephadm.wait_for_service:
+ service: rbd-mirror
+- cephadm.wait_for_service:
+ service: cephfs-mirror
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml
new file mode 100644
index 000000000..3f4964978
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml
@@ -0,0 +1,89 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rgw foorgw --port 8800
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}}
+
+- vip.exec:
+ host.a:
+ - dnf install -y python3-boto3 || apt install -y python3-boto3
+ - /home/ubuntu/cephtest/cephadm shell radosgw-admin user create --uid foouser --display-name foo > /tmp/user.json
+
+- python:
+ host.a: |
+ import boto3
+ import json
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ bucket.create()
+ bucket.put_object(Key='myobject', Body='thebody')
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create rgw --bucket foobucket --cluster-id foo --pseudo-path /foobucket
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/foobucket /mnt/foo
+ - find /mnt/foo -ls
+ - grep thebody /mnt/foo/myobject
+ - echo test > /mnt/foo/newobject
+ - sync
+
+- python:
+ host.a: |
+ import boto3
+ import json
+ from io import BytesIO
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ data = BytesIO()
+ bucket.download_fileobj(Fileobj=data, Key='newobject')
+ print(data.getvalue())
+ assert data.getvalue().decode() == 'test\n'
+
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export rm foo /foobucket
+ - ceph nfs cluster rm foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml
new file mode 100644
index 000000000..721aecfc3
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml
@@ -0,0 +1,90 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph orch apply rgw foorgw --port 8800
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}}
+
+- vip.exec:
+ host.a:
+ - dnf install -y python3-boto3 || apt install -y python3-boto3
+ - /home/ubuntu/cephtest/cephadm shell radosgw-admin user create --uid foouser --display-name foo > /tmp/user.json
+
+- python:
+ host.a: |
+ import boto3
+ import json
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ bucket.create()
+ bucket.put_object(Key='myobject', Body='thebody')
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create rgw --cluster-id foo --pseudo-path /foouser --user-id foouser
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/foouser /mnt/foo
+ - test -d /mnt/foo/foobucket
+ - find /mnt/foo -ls
+ - grep thebody /mnt/foo/foobucket/myobject
+ - echo test > /mnt/foo/foobucket/newobject
+ - sync
+
+- python:
+ host.a: |
+ import boto3
+ import json
+ from io import BytesIO
+
+ with open('/tmp/user.json', 'rt') as f:
+ info = json.loads(f.read())
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=info['keys'][0]['access_key'],
+ aws_secret_access_key=info['keys'][0]['secret_key'],
+ endpoint_url='http://localhost:8800',
+ )
+ bucket = s3.Bucket('foobucket')
+ data = BytesIO()
+ bucket.download_fileobj(Fileobj=data, Key='newobject')
+ print(data.getvalue())
+ assert data.getvalue().decode() == 'test\n'
+
+- vip.exec:
+ host.a:
+ - umount /mnt/foo
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export rm foo /foouser
+ - ceph nfs cluster rm foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml
new file mode 100644
index 000000000..b4e843df2
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml
@@ -0,0 +1,68 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+
+# deploy nfs + ingress
+- cephadm.apply:
+ specs:
+ - service_type: nfs
+ service_id: foo
+ placement:
+ count: 2
+ spec:
+ port: 12049
+ - service_type: ingress
+ service_id: nfs.foo
+ spec:
+ backend_service: nfs.foo
+ frontend_port: 2049
+ monitor_port: 9002
+ virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}"
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/fake /mnt/foo
+ - echo test > /mnt/foo/testfile
+ - sync
+
+# take each gateway down in turn and ensure things still work
+- cephadm.shell:
+ volumes:
+ - /mnt/foo:/mnt/foo
+ host.a:
+ - |
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ cat /mnt/foo/testfile
+ echo $haproxy > /mnt/foo/testfile
+ sync
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml
new file mode 100644
index 000000000..a47dd9d76
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml
@@ -0,0 +1,70 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph fs volume create foofs
+ - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}} --port 2999
+ - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake
+
+- cephadm.wait_for_service:
+ service: nfs.foo
+- cephadm.wait_for_service:
+ service: ingress.nfs.foo
+
+## export and mount
+
+- vip.exec:
+ host.a:
+ - mkdir /mnt/foo
+ - sleep 5
+ - mount -t nfs {{VIP0}}:/fake /mnt/foo -o port=2999
+ - echo test > /mnt/foo/testfile
+ - sync
+
+# take each gateway down in turn and ensure things still work
+- cephadm.shell:
+ volumes:
+ - /mnt/foo:/mnt/foo
+ host.a:
+ - |
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.nfs.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ cat /mnt/foo/testfile
+ echo $haproxy > /mnt/foo/testfile
+ sync
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
+
+# take each ganesha down in turn.
+# simulate "failure" by deleting the container
+- vip.exec:
+ all-hosts:
+ - |
+ echo "Check with $(hostname) ganesha(s) down..."
+ for c in `systemctl | grep ceph- | grep @nfs | awk '{print $1}'`; do
+ cid=`echo $c | sed 's/@/-/'`
+ id=`echo $c | cut -d @ -f 2 | sed 's/.service$//'`
+ fsid=`echo $c | cut -d @ -f 1 | cut -d - -f 2-`
+ echo "Removing daemon $id fsid $fsid..."
+ sudo $TESTDIR/cephadm rm-daemon --fsid $fsid --name $id
+
+ echo "Waking up cephadm..."
+ sudo $TESTDIR/cephadm shell -- ceph orch ps --refresh
+
+ while ! timeout 1 cat /mnt/foo/testfile ; do true ; done
+ echo "Mount is back!"
+ done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml
new file mode 100644
index 000000000..194f4e9de
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs.yaml
@@ -0,0 +1,13 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.apply:
+ specs:
+ - service_type: nfs
+ service_id: foo
+- cephadm.wait_for_service:
+ service: nfs.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml
new file mode 100644
index 000000000..959c5aa77
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs2.yaml
@@ -0,0 +1,12 @@
+tasks:
+
+# stop kernel nfs server, if running
+- vip.exec:
+ all-hosts:
+ - systemctl stop nfs-server
+
+- cephadm.shell:
+ host.a:
+ - ceph nfs cluster create foo
+- cephadm.wait_for_service:
+ service: nfs.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml
new file mode 100644
index 000000000..710edab73
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw-ingress.yaml
@@ -0,0 +1,60 @@
+tasks:
+- vip:
+
+# make sure cephadm notices the new IP
+- cephadm.shell:
+ host.a:
+ - ceph orch device ls --refresh
+
+# deploy rgw + ingress
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+ - service_type: ingress
+ service_id: rgw.foo
+ placement:
+ count: 2
+ spec:
+ backend_service: rgw.foo
+ frontend_port: 9000
+ monitor_port: 9001
+ virtual_ip: "{{VIP0}}/{{VIPPREFIXLEN}}"
+- cephadm.wait_for_service:
+ service: rgw.foo
+- cephadm.wait_for_service:
+ service: ingress.rgw.foo
+
+# take each component down in turn and ensure things still work
+- cephadm.shell:
+ host.a:
+ - |
+ echo "Check while healthy..."
+ curl http://{{VIP0}}:9000/
+
+ # stop each rgw in turn
+ echo "Check with each rgw stopped in turn..."
+ for rgw in `ceph orch ps | grep ^rgw.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $rgw
+ while ! ceph orch ps | grep $rgw | grep stopped; do sleep 1 ; done
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
+ ceph orch daemon start $rgw
+ while ! ceph orch ps | grep $rgw | grep running; do sleep 1 ; done
+ done
+
+ # stop each haproxy in turn
+ echo "Check with each haproxy down in turn..."
+ for haproxy in `ceph orch ps | grep ^haproxy.rgw.foo. | awk '{print $1}'`; do
+ ceph orch daemon stop $haproxy
+ while ! ceph orch ps | grep $haproxy | grep stopped; do sleep 1 ; done
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
+ ceph orch daemon start $haproxy
+ while ! ceph orch ps | grep $haproxy | grep running; do sleep 1 ; done
+ done
+
+ while ! curl http://{{VIP0}}:9000/ ; do sleep 1 ; done
diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml
new file mode 100644
index 000000000..cb2c6f4b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/rgw.yaml
@@ -0,0 +1,12 @@
+tasks:
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count_per_host: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+- cephadm.wait_for_service:
+ service: rgw.foo
diff --git a/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml b/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml
new file mode 100644
index 000000000..bb938848c
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-roleless/3-final.yaml
@@ -0,0 +1,10 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - stat -c '%u %g' /var/log/ceph | grep '167 167'
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls | grep '^osd.all-available-devices '
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/% b/qa/suites/orch/cephadm/smoke-singlehost/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/%
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/0-distro$ b/qa/suites/orch/cephadm/smoke-singlehost/0-distro$
new file mode 120000
index 000000000..d2dffb181
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/0-distro$
@@ -0,0 +1 @@
+../smoke/distro \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml b/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml
new file mode 100644
index 000000000..ca6019c66
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/1-start.yaml
@@ -0,0 +1,27 @@
+tasks:
+- cephadm:
+ roleless: true
+ single_host_defaults: true
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml b/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/2-services/basic.yaml
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml b/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml
new file mode 100644
index 000000000..cb2c6f4b6
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/2-services/rgw.yaml
@@ -0,0 +1,12 @@
+tasks:
+- cephadm.apply:
+ specs:
+ - service_type: rgw
+ service_id: foo
+ placement:
+ count_per_host: 4
+ host_pattern: "*"
+ spec:
+ rgw_frontend_port: 8000
+- cephadm.wait_for_service:
+ service: rgw.foo
diff --git a/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml b/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml
new file mode 100644
index 000000000..02f5b289c
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke-singlehost/3-final.yaml
@@ -0,0 +1,8 @@
+tasks:
+- cephadm.shell:
+ host.a:
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
diff --git a/qa/suites/orch/cephadm/smoke/% b/qa/suites/orch/cephadm/smoke/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/%
diff --git a/qa/suites/orch/cephadm/smoke/.qa b/qa/suites/orch/cephadm/smoke/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml b/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml
new file mode 120000
index 000000000..5206b6edd
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/0-nvme-loop.yaml
@@ -0,0 +1 @@
+.qa/overrides/nvme_loop.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/distro b/qa/suites/orch/cephadm/smoke/distro
new file mode 120000
index 000000000..be53a4833
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/distro
@@ -0,0 +1 @@
+.qa/distros/container-hosts/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/fixed-2.yaml b/qa/suites/orch/cephadm/smoke/fixed-2.yaml
new file mode 100644
index 000000000..e93564aa9
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/fixed-2.yaml
@@ -0,0 +1,32 @@
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+ - ceph.rgw.foo.a
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+ - ceph.iscsi.iscsi.a
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/smoke/mon_election b/qa/suites/orch/cephadm/smoke/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/smoke/start.yaml b/qa/suites/orch/cephadm/smoke/start.yaml
new file mode 100644
index 000000000..77f493ca1
--- /dev/null
+++ b/qa/suites/orch/cephadm/smoke/start.yaml
@@ -0,0 +1,16 @@
+tasks:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
+- cephadm.shell:
+ mon.a:
+ - stat -c '%u %g' /var/log/ceph | grep '167 167'
+ - ceph orch status
+ - ceph orch ps
+ - ceph orch ls
+ - ceph orch host ls
+ - ceph orch device ls
+ - ceph orch ls --format yaml
+ - ceph orch ls | grep '^osd '
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/% b/qa/suites/orch/cephadm/thrash-old-clients/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/%
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/.qa b/qa/suites/orch/cephadm/thrash-old-clients/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml
new file mode 120000
index 000000000..5393a7554
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/2-size-2-min-size.yaml
@@ -0,0 +1 @@
+.qa/overrides/2-size-2-min-size.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml
new file mode 120000
index 000000000..5ff70eadf
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/0-size-min-size-overrides/3-size-2-min-size.yaml
@@ -0,0 +1 @@
+.qa/overrides/3-size-2-min-size.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa b/qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml
new file mode 100644
index 000000000..f29bb3527
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous-v1only.yaml
@@ -0,0 +1,32 @@
+overrides:
+ ceph:
+ mon_bind_msgr2: false
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ ms type: async
+ ms bind msgr2: false
+tasks:
+- install:
+ branch: luminous
+ exclude_packages:
+ - librados3
+ - ceph-mgr-dashboard
+ - ceph-mgr-diskprediction-local
+ - ceph-mgr-rook
+ - ceph-mgr-cephadm
+ - cephadm
+ - ceph-immutable-object-cache
+ - ceph-base
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ - librados-devel
+ extra_packages:
+ - librados2
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml
new file mode 100644
index 000000000..f51a883fa
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/luminous.yaml
@@ -0,0 +1,31 @@
+overrides:
+ ceph:
+ mon_bind_msgr2: false
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ mon warn on msgr2 not enabled: false
+tasks:
+- install:
+ branch: luminous
+ exclude_packages:
+ - librados3
+ - ceph-mgr-dashboard
+ - ceph-mgr-diskprediction-local
+ - ceph-mgr-rook
+ - ceph-mgr-cephadm
+ - cephadm
+ - ceph-immutable-object-cache
+ - ceph-base
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ - librados-devel
+ extra_packages:
+ - librados2
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml
new file mode 100644
index 000000000..a37b451e5
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic-v1only.yaml
@@ -0,0 +1,32 @@
+overrides:
+ ceph:
+ mon_bind_msgr2: false
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ ms type: async
+ ms bind msgr2: false
+tasks:
+- install:
+ branch: mimic
+ exclude_packages:
+ - librados3
+ - ceph-mgr-dashboard
+ - ceph-mgr-diskprediction-local
+ - ceph-mgr-rook
+ - ceph-mgr-cephadm
+ - cephadm
+ - ceph-immutable-object-cache
+ - ceph-base
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ - librados-devel
+ extra_packages:
+ - librados2
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml
new file mode 100644
index 000000000..07aefd15c
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/mimic.yaml
@@ -0,0 +1,32 @@
+overrides:
+ ceph:
+ mon_bind_msgr2: false
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ ms type: async
+ mon warn on msgr2 not enabled: false
+tasks:
+- install:
+ branch: mimic
+ exclude_packages:
+ - librados3
+ - ceph-mgr-dashboard
+ - ceph-mgr-diskprediction-local
+ - ceph-mgr-rook
+ - ceph-mgr-cephadm
+ - cephadm
+ - ceph-immutable-object-cache
+ - ceph-base
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ - librados-devel
+ extra_packages:
+ - librados2
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml
new file mode 100644
index 000000000..68cc30d47
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v1only.yaml
@@ -0,0 +1,25 @@
+overrides:
+ ceph:
+ mon_bind_msgr2: false
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ ms type: async
+ ms bind msgr2: false
+tasks:
+- install:
+ branch: nautilus
+ exclude_packages:
+ - cephadm
+ - ceph-mgr-cephadm
+ - ceph-immutable-object-cache
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ extra_packages:
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml
new file mode 100644
index 000000000..8e3f2956a
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus-v2only.yaml
@@ -0,0 +1,25 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - \(MON_DOWN\)
+ conf:
+ global:
+ ms type: async
+ ms bind msgr2: true
+ ms bind msgr1: false
+tasks:
+- install:
+ branch: nautilus
+ exclude_packages:
+ - cephadm
+ - ceph-mgr-cephadm
+ - ceph-immutable-object-cache
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ extra_packages:
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml
new file mode 100644
index 000000000..c1aca646f
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/nautilus.yaml
@@ -0,0 +1,20 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - \(MON_DOWN\)
+tasks:
+- install:
+ branch: nautilus
+ exclude_packages:
+ - cephadm
+ - ceph-mgr-cephadm
+ - ceph-immutable-object-cache
+ - python3-rados
+ - python3-rgw
+ - python3-rbd
+ - python3-cephfs
+ extra_packages:
+ - python-rados
+ - python-rgw
+ - python-rbd
+ - python-cephfs
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml b/qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml
new file mode 100644
index 000000000..a7ea6239d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/1-install/octopus.yaml
@@ -0,0 +1,34 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - \(MON_DOWN\)
+tasks:
+- install:
+ branch: octopus
+ exclude_packages:
+ - ceph-mgr-dashboard
+ - ceph-mgr-diskprediction-local
+ - ceph-mgr-rook
+ - ceph-mgr-cephadm
+ - ceph-base-debuginfo
+ - ceph-common-debuginfo
+ - ceph-immutable-object-cache-debuginfo
+ - ceph-radosgw-debuginfo
+ - ceph-test-debuginfo
+ - ceph-base-debuginfo
+ - ceph-mgr-debuginfo
+ - ceph-mds-debuginfo
+ - ceph-mon-debuginfo
+ - ceph-osd-debuginfo
+ - ceph-fuse-debuginfo
+ - librados-devel-debuginfo
+ - libcephfs2-debuginfo
+ - librados2-debuginfo
+ - librbd1-debuginfo
+ - python3-cephfs-debuginfo
+ - python3-rados-debuginfo
+ - python3-rbd-debuginfo
+ - python3-rgw-debuginfo
+ - rbd-fuse-debuginfo
+ - rbd-mirror-debuginfo
+ - rbd-nbd-debuginfo
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa b/qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/backoff/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml b/qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/backoff/normal.yaml
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml b/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml
new file mode 100644
index 000000000..66d06117e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd backoff on peering: true
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml b/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml
new file mode 100644
index 000000000..e61099065
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/backoff/peering_and_degraded.yaml
@@ -0,0 +1,6 @@
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd backoff on peering: true
+ osd backoff on degraded: true
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml b/qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml
new file mode 100644
index 000000000..016ce36da
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/ceph.yaml
@@ -0,0 +1,7 @@
+# Don't verify os + flavor + sha1
+verify_ceph_hash: false
+tasks:
+- cephadm:
+ conf:
+ mon:
+ auth allow insecure global id reclaim: true
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/clusters/+ b/qa/suites/orch/cephadm/thrash-old-clients/clusters/+
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/clusters/+
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa b/qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/clusters/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml b/qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml
new file mode 100644
index 000000000..b0f3b9b4d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/clusters/openstack.yaml
@@ -0,0 +1,4 @@
+openstack:
+ - volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml b/qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml
new file mode 100644
index 000000000..9af5382ea
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/clusters/three-plus-one.yaml
@@ -0,0 +1,13 @@
+roles:
+- [mon.a, mgr.y, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mgr.x, osd.4, osd.5, osd.6, osd.7, client.1]
+- [mon.c, osd.8, osd.9, osd.10, osd.11, client.2]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml
new file mode 100644
index 000000000..aa867660d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/crush-compat.yaml
@@ -0,0 +1,6 @@
+tasks:
+- exec:
+ mon.a:
+ - while ! ceph balancer status ; do sleep 1 ; done
+ - ceph balancer mode crush-compat
+ - ceph balancer on
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/d-balancer/on.yaml
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa b/qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/distro$/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml b/qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml
new file mode 120000
index 000000000..cfb85f10e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/distro$/ubuntu_18.04.yaml
@@ -0,0 +1 @@
+.qa/distros/all/ubuntu_18.04.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/mon_election b/qa/suites/orch/cephadm/thrash-old-clients/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml
new file mode 100644
index 000000000..ec45f8882
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/fastclose.yaml
@@ -0,0 +1,9 @@
+overrides:
+ ceph:
+ conf:
+ global:
+ ms inject socket failures: 2500
+ ms tcp read timeout: 5
+ mon client directed command retry: 5
+ log-ignorelist:
+ - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml
new file mode 100644
index 000000000..cc9a3ae69
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/few.yaml
@@ -0,0 +1,10 @@
+overrides:
+ ceph:
+ conf:
+ global:
+ ms inject socket failures: 5000
+ mon client directed command retry: 5
+ osd:
+ osd heartbeat use min delay socket: true
+ log-ignorelist:
+ - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml
new file mode 100644
index 000000000..d7cec6f36
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/msgr-failures/osd-delay.yaml
@@ -0,0 +1,12 @@
+overrides:
+ ceph:
+ conf:
+ global:
+ ms inject socket failures: 2500
+ ms inject delay type: osd
+ ms inject delay probability: .005
+ ms inject delay max: 1
+ ms inject internal delays: .002
+ mon client directed command retry: 5
+ log-ignorelist:
+ - \(OSD_SLOW_PING_TIME
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/rados.yaml b/qa/suites/orch/cephadm/thrash-old-clients/rados.yaml
new file mode 120000
index 000000000..d256979c0
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/rados.yaml
@@ -0,0 +1 @@
+.qa/config/rados.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml
new file mode 100644
index 000000000..cc232ab88
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/careful.yaml
@@ -0,0 +1,25 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ conf:
+ osd:
+ osd debug reject backfill probability: .3
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd max backfills: 3
+ osd snap trim sleep: 2
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 1
+ chance_pgpnum_fix: 1
+ aggressive_pg_num_changes: false
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml
new file mode 100644
index 000000000..c04f9535c
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/default.yaml
@@ -0,0 +1,24 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ conf:
+ osd:
+ osd debug reject backfill probability: .3
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd max backfills: 3
+ osd snap trim sleep: 2
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 1
+ chance_pgpnum_fix: 1
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml
new file mode 100644
index 000000000..27881d218
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/mapgap.yaml
@@ -0,0 +1,26 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ - osd_map_cache_size
+ conf:
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+ osd:
+ osd map cache size: 1
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd scrub during recovery: false
+ osd max backfills: 6
+tasks:
+- thrashosds:
+ timeout: 1800
+ chance_pgnum_grow: 0.25
+ chance_pgpnum_fix: 0.25
+ chance_test_map_discontinuity: 2
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml
new file mode 100644
index 000000000..f18a88711
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/morepggrow.yaml
@@ -0,0 +1,22 @@
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ journal throttle high multiple: 2
+ journal throttle max multiple: 10
+ filestore queue throttle high multiple: 2
+ filestore queue throttle max multiple: 10
+ osd max backfills: 9
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 3
+ chance_pgpnum_fix: 1
+openstack:
+- volumes:
+ size: 50
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/none.yaml
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml
new file mode 100644
index 000000000..54498d0cf
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashers/pggrow.yaml
@@ -0,0 +1,24 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ conf:
+ osd:
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ filestore odsync write: true
+ osd max backfills: 2
+ osd snap trim sleep: .5
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 2
+ chance_pgpnum_fix: 1
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml b/qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml
new file mode 120000
index 000000000..9124eb1aa
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/thrashosds-health.yaml
@@ -0,0 +1 @@
+.qa/tasks/thrashosds-health.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa b/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml b/qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml
new file mode 100644
index 000000000..33f667ffd
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/cache-snaps.yaml
@@ -0,0 +1,34 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - must scrub before tier agent can activate
+tasks:
+- exec:
+ client.0:
+ - sudo ceph osd pool create base 4
+ - sudo ceph osd pool application enable base rados
+ - sudo ceph osd pool create cache 4
+ - sudo ceph osd tier add base cache
+ - sudo ceph osd tier cache-mode cache writeback
+ - sudo ceph osd tier set-overlay base cache
+ - sudo ceph osd pool set cache hit_set_type bloom
+ - sudo ceph osd pool set cache hit_set_count 8
+ - sudo ceph osd pool set cache hit_set_period 3600
+ - sudo ceph osd pool set cache target_max_objects 250
+ - sudo ceph osd pool set cache min_read_recency_for_promote 2
+- rados:
+ clients: [client.2]
+ pools: [base]
+ ops: 4000
+ objects: 500
+ op_weights:
+ read: 100
+ write: 100
+ delete: 50
+ copy_from: 50
+ cache_flush: 50
+ cache_try_flush: 50
+ cache_evict: 50
+ snap_create: 50
+ snap_remove: 50
+ rollback: 50
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml b/qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml
new file mode 100644
index 000000000..6a89a4e6e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/radosbench.yaml
@@ -0,0 +1,33 @@
+overrides:
+ ceph:
+ conf:
+ client.2:
+ debug ms: 1
+ debug objecter: 20
+ debug rados: 20
+tasks:
+- full_sequential:
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
+ - radosbench:
+ clients: [client.2]
+ time: 90
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml b/qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml
new file mode 100644
index 000000000..a8bbbafec
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/rbd_cls.yaml
@@ -0,0 +1,7 @@
+meta:
+- desc: |
+ rbd object class functional tests
+tasks:
+- exec:
+ client.2:
+ - ceph_test_cls_rbd --gtest_filter=-TestClsRbd.get_features:TestClsRbd.parents:TestClsRbd.mirror
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml b/qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml
new file mode 100644
index 000000000..f0a5735a9
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/snaps-few-objects.yaml
@@ -0,0 +1,13 @@
+tasks:
+- rados:
+ clients: [client.2]
+ ops: 4000
+ objects: 50
+ op_weights:
+ read: 100
+ write: 100
+ delete: 50
+ snap_create: 50
+ snap_remove: 50
+ rollback: 50
+ copy_from: 50
diff --git a/qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml b/qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml
new file mode 100644
index 000000000..39617b37e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash-old-clients/workloads/test_rbd_api.yaml
@@ -0,0 +1,8 @@
+meta:
+- desc: |
+ librbd C and C++ api tests
+workload:
+- workunit:
+ clients:
+ client.2:
+ - rbd/test_librbd.sh
diff --git a/qa/suites/orch/cephadm/thrash/% b/qa/suites/orch/cephadm/thrash/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/%
diff --git a/qa/suites/orch/cephadm/thrash/.qa b/qa/suites/orch/cephadm/thrash/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/0-distro b/qa/suites/orch/cephadm/thrash/0-distro
new file mode 120000
index 000000000..f0db8bb61
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/0-distro
@@ -0,0 +1 @@
+../workunits/0-distro \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/1-start.yaml b/qa/suites/orch/cephadm/thrash/1-start.yaml
new file mode 100644
index 000000000..a1b89e44d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/1-start.yaml
@@ -0,0 +1,7 @@
+tasks:
+- install:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
diff --git a/qa/suites/orch/cephadm/thrash/2-thrash.yaml b/qa/suites/orch/cephadm/thrash/2-thrash.yaml
new file mode 100644
index 000000000..05e0f8e76
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/2-thrash.yaml
@@ -0,0 +1,26 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - but it is still running
+ - objects unfound and apparently lost
+ conf:
+ osd:
+ osd debug reject backfill probability: .3
+ osd scrub min interval: 60
+ osd scrub max interval: 120
+ osd max backfills: 3
+ osd snap trim sleep: 2
+ osd delete sleep: 1
+ mon:
+ mon min osdmap epochs: 50
+ paxos service trim min: 10
+ # prune full osdmaps regularly
+ mon osdmap full prune min: 15
+ mon osdmap full prune interval: 2
+ mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+ timeout: 1200
+ chance_pgnum_grow: 1
+ chance_pgnum_shrink: 1
+ chance_pgpnum_fix: 1
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/.qa b/qa/suites/orch/cephadm/thrash/3-tasks/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml
new file mode 120000
index 000000000..34e657e04
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/rados_api_tests.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/rados_api_tests.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml
new file mode 120000
index 000000000..dad17e0de
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/radosbench.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/radosbench.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml
new file mode 120000
index 000000000..6aa66aa37
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/small-objects.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/small-objects.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml b/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml
new file mode 120000
index 000000000..c9cc4cd3e
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/3-tasks/snaps-few-objects.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/thrash/workloads/snaps-few-objects.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/fixed-2.yaml b/qa/suites/orch/cephadm/thrash/fixed-2.yaml
new file mode 120000
index 000000000..5c3e0593c
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/fixed-2.yaml
@@ -0,0 +1 @@
+../smoke/fixed-2.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/msgr b/qa/suites/orch/cephadm/thrash/msgr
new file mode 120000
index 000000000..57bee80db
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/msgr
@@ -0,0 +1 @@
+.qa/msgr \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/thrash/root.yaml b/qa/suites/orch/cephadm/thrash/root.yaml
new file mode 100644
index 000000000..bedb31d5d
--- /dev/null
+++ b/qa/suites/orch/cephadm/thrash/root.yaml
@@ -0,0 +1,3 @@
+overrides:
+ cephadm:
+ cephadm_mode: root
diff --git a/qa/suites/orch/cephadm/upgrade/% b/qa/suites/orch/cephadm/upgrade/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/%
diff --git a/qa/suites/orch/cephadm/upgrade/.qa b/qa/suites/orch/cephadm/upgrade/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
new file mode 100644
index 000000000..a1a9cb5ef
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-centos_8.stream_container-tools.yaml
@@ -0,0 +1,43 @@
+os_type: centos
+os_version: "8.stream"
+
+tasks:
+- pexec:
+ all:
+ - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
+ - sudo dnf -y module reset container-tools
+ - sudo dnf -y module install container-tools
+ - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
+- cephadm:
+ image: docker.io/ceph/ceph:v15.2.0
+ cephadm_branch: v15.2.0
+ cephadm_git_url: https://github.com/ceph/ceph
+ # avoid --cap-add=PTRACE + --privileged for older cephadm versions
+ allow_ptrace: false
+ # deploy additional mons the "old" (octopus) way
+ add_mons_via_daemon_add: true
+ avoid_pacific_features: true
+
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+# - ceph.rgw.realm.zone.a # Disabled, needs 15.2.4 as an upgrade start
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+# - ceph.iscsi.iscsi.a # needs later start point
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml
new file mode 100644
index 000000000..fde68b32a
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04-15.2.9.yaml
@@ -0,0 +1,37 @@
+os_type: ubuntu
+os_version: "20.04"
+
+tasks:
+- cephadm:
+ image: docker.io/ceph/ceph:v15.2.9
+ cephadm_branch: v15.2.9
+ cephadm_git_url: https://github.com/ceph/ceph
+ # avoid --cap-add=PTRACE + --privileged for older cephadm versions
+ allow_ptrace: false
+ # deploy additional mons the "old" (octopus) way
+ add_mons_via_daemon_add: true
+ avoid_pacific_features: true
+
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+# - ceph.rgw.realm.zone.a # CLI change in v16 pacific
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+ - ceph.iscsi.iscsi.a
diff --git a/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml
new file mode 100644
index 000000000..983b4f3dc
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/1-start-distro/1-start-ubuntu_20.04.yaml
@@ -0,0 +1,37 @@
+os_type: ubuntu
+os_version: "20.04"
+
+tasks:
+- cephadm:
+ image: docker.io/ceph/ceph:v15.2.0
+ cephadm_branch: v15.2.0
+ cephadm_git_url: https://github.com/ceph/ceph
+ # avoid --cap-add=PTRACE + --privileged for older cephadm versions
+ allow_ptrace: false
+ # deploy additional mons the "old" (octopus) way
+ add_mons_via_daemon_add: true
+ avoid_pacific_features: true
+
+roles:
+- - mon.a
+ - mon.c
+ - mgr.y
+ - osd.0
+ - osd.1
+ - osd.2
+ - osd.3
+ - client.0
+# - ceph.rgw.realm.zone.a # Disabled, needs 15.2.4 as an upgrade start
+ - node-exporter.a
+ - alertmanager.a
+- - mon.b
+ - mgr.x
+ - osd.4
+ - osd.5
+ - osd.6
+ - osd.7
+ - client.1
+ - prometheus.a
+ - grafana.a
+ - node-exporter.b
+# - ceph.iscsi.iscsi.a # needs later start point
diff --git a/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml b/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/2-repo_digest/defaut.yaml
diff --git a/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml b/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml
new file mode 100644
index 000000000..2e6bbfd92
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/2-repo_digest/repo_digest.yaml
@@ -0,0 +1,4 @@
+tasks:
+- cephadm.shell:
+ mon.a:
+ - ceph config set mgr mgr/cephadm/use_repo_digest false --force
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa b/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml
new file mode 100644
index 000000000..80463a6c0
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/simple.yaml
@@ -0,0 +1,13 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ - radosgw-admin realm create --rgw-realm=r --default
+ - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
+ - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
+ - radosgw-admin period update --rgw-realm=r --commit
+ - ceph orch apply rgw r z --placement=2 --port=8000
+ - sleep 120
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
diff --git a/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml
new file mode 100644
index 000000000..ebd06c92d
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/3-upgrade/staggered.yaml
@@ -0,0 +1,111 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ - radosgw-admin realm create --rgw-realm=r --default
+ - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
+ - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
+ - radosgw-admin period update --rgw-realm=r --commit
+ - ceph orch apply rgw r z --placement=2 --port=8000
+ - sleep 180
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+ - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+ - ceph config set global log_to_journald false --force
+ # get some good info on the state of things pre-upgrade. Useful for debugging
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ - ceph orch ls
+ # doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code
+ # until there is a stable version that contains it, we can test by manually upgrading a mgr daemon
+ - ceph config set mgr container_image quay.ceph.io/ceph-ci/ceph:$sha1
+ - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)"
+ - ceph orch ps --refresh
+ - sleep 180
+ # gather more possible debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ # check that there are two different versions found for mgr daemon (which implies we upgraded one)
+ - ceph versions | jq -e '.mgr | length == 2'
+ - ceph mgr fail
+ - sleep 180
+ # now try upgrading the other mgr
+ # we should now have access to --image flag for the daemon redeploy command
+ - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
+ - ceph orch ps --refresh
+ - sleep 180
+ # gather more possible debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ - ceph mgr fail
+ - sleep 180
+ # gather more debugging info
+ - ceph orch ps
+ - ceph versions
+ - ceph -s
+ # now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs
+ - ceph versions | jq -e '.mgr | length == 1'
+ - ceph mgr fail
+ - sleep 180
+ # debugging info
+ - ceph orch ps
+ - ceph versions
+ # to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on new new version
+ # also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ # verify only one version found for mgrs and that their version hash matches what we are upgrading to
+ - ceph versions | jq -e '.mgr | length == 1'
+ - ceph versions | jq -e '.mgr | keys' | grep $sha1
+ # verify overall we still se two versions, basically to make sure --daemon-types wans't ignored and all daemons upgraded
+ - ceph versions | jq -e '.overall | length == 2'
+ # check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
+ # upgrade only the mons on one of the two hosts
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify tow different version seen for mons
+ - ceph versions | jq -e '.mon | length == 2'
+ # upgrade mons on the other hosts
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all mons now on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.mon | length == 1'
+ - ceph versions | jq -e '.mon | keys' | grep $sha1
+ # verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
+ # upgrade exactly 2 osd daemons
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify two different versions now seen for osds
+ - ceph versions | jq -e '.osd | length == 2'
+ # verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds)
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
+ # upgrade one more osd
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions | jq -e '.osd | length == 2'
+ # verify now 8 daemons have been upgraded
+ - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
+ # upgrade the rest of the osds
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all osds are now on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.osd | length == 1'
+ - ceph versions | jq -e '.osd | keys' | grep $sha1
+ # upgrade the rgw daemons using --services
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.r.z
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ # verify all rgw daemons on same version and version hash matches what we are upgrading to
+ - ceph versions | jq -e '.rgw | length == 1'
+ - ceph versions | jq -e '.rgw | keys' | grep $sha1
+ # run upgrade one more time with no filter parameters to make sure anything left gets upgraded
+ - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
diff --git a/qa/suites/orch/cephadm/upgrade/4-wait.yaml b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
new file mode 100644
index 000000000..58afe00c5
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/4-wait.yaml
@@ -0,0 +1,14 @@
+tasks:
+- cephadm.shell:
+ env: [sha1]
+ mon.a:
+ - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
+ - ceph orch ps
+ - ceph versions
+ - echo "wait for servicemap items w/ changing names to refresh"
+ - sleep 60
+ - ceph orch ps
+ - ceph versions
+ - ceph versions | jq -e '.overall | length == 1'
+ - ceph versions | jq -e '.overall | keys' | grep $sha1
+ - ceph orch ls | grep '^osd '
diff --git a/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml b/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml
new file mode 100644
index 000000000..236ea6c6b
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/5-upgrade-ls.yaml
@@ -0,0 +1,6 @@
+tasks:
+- cephadm.shell:
+ mon.a:
+ - ceph orch upgrade ls
+ - ceph orch upgrade ls --image quay.io/ceph/ceph | grep 16.2.0
+ - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2
diff --git a/qa/suites/orch/cephadm/upgrade/mon_election b/qa/suites/orch/cephadm/upgrade/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/upgrade/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/% b/qa/suites/orch/cephadm/with-work/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/%
diff --git a/qa/suites/orch/cephadm/with-work/.qa b/qa/suites/orch/cephadm/with-work/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/0-distro b/qa/suites/orch/cephadm/with-work/0-distro
new file mode 120000
index 000000000..92c9153ea
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/0-distro
@@ -0,0 +1 @@
+../smoke/distro/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/fixed-2.yaml b/qa/suites/orch/cephadm/with-work/fixed-2.yaml
new file mode 120000
index 000000000..5c3e0593c
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/fixed-2.yaml
@@ -0,0 +1 @@
+../smoke/fixed-2.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/mode/.qa b/qa/suites/orch/cephadm/with-work/mode/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/mode/packaged.yaml b/qa/suites/orch/cephadm/with-work/mode/packaged.yaml
new file mode 100644
index 000000000..ba8d43218
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/packaged.yaml
@@ -0,0 +1,5 @@
+overrides:
+ cephadm:
+ cephadm_mode: cephadm-package
+ install:
+ extra_packages: [cephadm]
diff --git a/qa/suites/orch/cephadm/with-work/mode/root.yaml b/qa/suites/orch/cephadm/with-work/mode/root.yaml
new file mode 100644
index 000000000..bedb31d5d
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mode/root.yaml
@@ -0,0 +1,3 @@
+overrides:
+ cephadm:
+ cephadm_mode: root
diff --git a/qa/suites/orch/cephadm/with-work/mon_election b/qa/suites/orch/cephadm/with-work/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/msgr b/qa/suites/orch/cephadm/with-work/msgr
new file mode 120000
index 000000000..57bee80db
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/msgr
@@ -0,0 +1 @@
+.qa/msgr \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/start.yaml b/qa/suites/orch/cephadm/with-work/start.yaml
new file mode 100644
index 000000000..a1b89e44d
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/start.yaml
@@ -0,0 +1,7 @@
+tasks:
+- install:
+- cephadm:
+ conf:
+ mgr:
+ debug ms: 1
+ debug mgr: 20
diff --git a/qa/suites/orch/cephadm/with-work/tasks/.qa b/qa/suites/orch/cephadm/with-work/tasks/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml b/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml
new file mode 120000
index 000000000..2ce80f969
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/rados_api_tests.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/basic/tasks/rados_api_tests.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml b/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml
new file mode 120000
index 000000000..210ad8f18
--- /dev/null
+++ b/qa/suites/orch/cephadm/with-work/tasks/rados_python.yaml
@@ -0,0 +1 @@
+.qa/suites/rados/basic/tasks/rados_python.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/% b/qa/suites/orch/cephadm/workunits/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/%
diff --git a/qa/suites/orch/cephadm/workunits/.qa b/qa/suites/orch/cephadm/workunits/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/0-distro/.qa b/qa/suites/orch/cephadm/workunits/0-distro/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/0-distro/.qa
@@ -0,0 +1 @@
+../.qa \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/0-distro/centos_8.stream_container_tools.yaml b/qa/suites/orch/cephadm/workunits/0-distro/centos_8.stream_container_tools.yaml
new file mode 120000
index 000000000..7a86f967f
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/0-distro/centos_8.stream_container_tools.yaml
@@ -0,0 +1 @@
+.qa/distros/podman/centos_8.stream_container_tools.yaml \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/mon_election b/qa/suites/orch/cephadm/workunits/mon_election
new file mode 120000
index 000000000..3f331e621
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/mon_election
@@ -0,0 +1 @@
+.qa/mon_election \ No newline at end of file
diff --git a/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml b/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml
new file mode 100644
index 000000000..e04fc1eea
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_adoption.yaml
@@ -0,0 +1,11 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- install:
+- exec:
+ mon.a:
+ - yum install -y python3 || apt install -y python3
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_adoption.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml b/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml
new file mode 100644
index 000000000..4d253517c
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_cephadm.yaml
@@ -0,0 +1,11 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- install:
+- exec:
+ mon.a:
+ - yum install -y python3 || apt install -y python3
+- workunit:
+ clients:
+ client.0:
+ - cephadm/test_cephadm.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml b/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml
new file mode 100644
index 000000000..4a1ac88de
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_cephadm_repos.yaml
@@ -0,0 +1,8 @@
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+tasks:
+- workunit:
+ no_coverage_and_limits: true
+ clients:
+ client.0:
+ - cephadm/test_repos.sh
diff --git a/qa/suites/orch/cephadm/workunits/task/test_nfs.yaml b/qa/suites/orch/cephadm/workunits/task/test_nfs.yaml
new file mode 100644
index 000000000..8448c1a2f
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_nfs.yaml
@@ -0,0 +1,17 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply mds a
+- cephfs_test_runner:
+ modules:
+ - tasks.cephfs.test_nfs
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml b/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml
new file mode 100644
index 000000000..ec65fb116
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_orch_cli.yaml
@@ -0,0 +1,17 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply mds a
+- cephfs_test_runner:
+ modules:
+ - tasks.cephadm_cases.test_cli
diff --git a/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
new file mode 100644
index 000000000..2a33dc839
--- /dev/null
+++ b/qa/suites/orch/cephadm/workunits/task/test_orch_cli_mon.yaml
@@ -0,0 +1,45 @@
+roles:
+- - host.a
+ - osd.0
+ - osd.1
+ - osd.2
+ - mon.a
+ - mgr.a
+ - client.0
+- - host.b
+ - osd.3
+ - osd.4
+ - osd.5
+ - mon.b
+ - mgr.b
+ - client.1
+- - host.c
+ - osd.6
+ - osd.7
+ - osd.8
+ - mon.c
+ - mgr.c
+ - client.2
+- - host.d
+ - osd.9
+ - osd.10
+ - osd.11
+ - mon.d
+ - mgr.d
+ - client.3
+- - host.e
+ - osd.12
+ - osd.13
+ - osd.14
+ - mon.e
+ - mgr.e
+ - client.4
+tasks:
+- install:
+- cephadm:
+- cephadm.shell:
+ host.a:
+ - ceph orch apply mds a
+- cephfs_test_runner:
+ modules:
+ - tasks.cephadm_cases.test_cli_mon