12 files changed, 119 insertions, 0 deletions
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/% b/qa/suites/upgrade/quincy-x/filestore-remove-check/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/%
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa
new file mode 120000
index 000000000..fea2489fd
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/.qa
@@ -0,0 +1 @@
+../.qa
+\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/+ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/+
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/+
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/.qa
@@ -0,0 +1 @@
+../.qa/
+\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml
new file mode 100644
index 000000000..5caffc353
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/openstack.yaml
@@ -0,0 +1,6 @@
+openstack:
+  - machine:
+      disk: 100 # GB
+  - volumes: # attached to each instance
+      count: 4
+      size: 30 # GB
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml
new file mode 100644
index 000000000..b4b6f4d90
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/0-cluster/start.yaml
@@ -0,0 +1,34 @@
+meta:
+- desc: |
+   Run ceph on one nodes,
+   Use xfs beneath the osds. upgrade to reef
+   should fail to start the osds with filestore
+overrides:
+  ceph:
+    mon_bind_msgr2: false
+    mon_bind_addrvec: false
+    mon-health-to-clog: false
+    wait-for-healthy: false
+    wait-for-osds-up: false
+    wait-for-scrub: false
+    skip_stop_pg_num_changes: true
+    fs: xfs
+    log-ignorelist:
+      - overall HEALTH_
+      - \(MON_DOWN\)
+      - \(MGR_DOWN\)
+      - slow request
+      - \(MON_MSGR2_NOT_ENABLED\)
+      - \(POOL_APP_NOT_ENABLED\)
+    conf:
+      global:
+        enable experimental unrecoverable data corrupting features: "*"
+        mon warn on msgr2 not enabled: false
+      mon:
+        mon warn on osd down out interval zero: false
+roles:
+- - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - osd.2
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/.qa
@@ -0,0 +1 @@
+../.qa/
+\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml
new file mode 100644
index 000000000..471bd61df
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/1-ceph-install/quincy.yaml
@@ -0,0 +1,32 @@
+meta:
+- desc: install ceph/quincy latest
+tasks:
+- install:
+    exclude_packages:
+      - ceph-mgr-cephadm
+      - cephadm
+      - libcephfs-dev
+    branch: quincy
+- print: "**** done install quincy"
+- ceph:
+    create_rbd_pool: false
+    conf:
+      global:
+        bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
+        mon pg warn min per osd: 0
+      mon:
+        mon_warn_on_insecure_global_id_reclaim: false
+        mon_warn_on_insecure_global_id_reclaim_allowed: false
+    log-ignorelist:
+      - Not found or unloadable
+      - evicting unresponsive client
+- exec:
+    osd.0:
+      - ceph osd require-osd-release quincy
+- print: "**** done ceph"
+overrides:
+  ceph:
+    conf:
+      mon:
+        mon warn on osd down out interval zero: false
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml
new file mode 100644
index 000000000..6aa429f18
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/2 - upgrade.yaml
@@ -0,0 +1,20 @@
+meta:
+- desc: |
+   install upgrade ceph/-x on cluster
+   restart : mons, osd.*
+tasks:
+- install.upgrade:
+    mon.a:
+- exec:
+    osd.0:
+      - ceph osd require-osd-release quincy
+- print: "**** done install.upgrade of nodes"
+- ceph.restart:
+    daemons: [mon.a,mgr.x,osd.0,osd.1,osd.2]
+    mon-health-to-clog: false
+    wait-for-healthy: false
+    wait-for-osds-up: false
+    wait-for-scrub: false
+    skip_stop_pg_num_changes: true
+    expected-failure: "FileStore has been deprecated and is no longer supported"
+- print: "**** done ceph.restart of all mons and osds"
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/.qa
@@ -0,0 +1 @@
+../.qa/
+\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml
new file mode 100644
index 000000000..b6ef47b06
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/objectstore/filestore-xfs.yaml
@@ -0,0 +1,14 @@
+overrides:
+  ceph:
+    fs: xfs
+    conf:
+      osd:
+        osd objectstore: filestore
+        osd sloppy crc: true
+  ceph-deploy:
+    fs: xfs
+    filestore: True
+    conf:
+      osd:
+        osd objectstore: filestore
+        osd sloppy crc: true
+\ No newline at end of file
diff --git a/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml b/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml
new file mode 100644
index 000000000..e1374c410
--- /dev/null
+++ b/qa/suites/upgrade/quincy-x/filestore-remove-check/ubuntu_20.04.yaml
@@ -0,0 +1,9 @@
+os_type: ubuntu
+os_version: "20.04"
+# the normal ubuntu 20.04 kernel (5.4.0-88-generic currently) have a bug that prevents the nvme_loop
+# from behaving.  I think it is this:
+#   https://lkml.org/lkml/2020/9/21/1456
+# (at least, that is the symptom: nvme nvme1: Connect command failed, error wo/DNR bit: 880)
+overrides:
+  kernel:
+    hwe: true
+\ No newline at end of file