summaryrefslogtreecommitdiffstats
path: root/qa/cephfs
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:54:28 +0000
commite6918187568dbd01842d8d1d2c808ce16a894239 (patch)
tree64f88b554b444a49f656b6c656111a145cbbaa28 /qa/cephfs
parentInitial commit. (diff)
downloadceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz
ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'qa/cephfs')
l---------qa/cephfs/.qa1
-rw-r--r--qa/cephfs/begin/+0
-rw-r--r--qa/cephfs/begin/0-install.yaml65
-rw-r--r--qa/cephfs/begin/1-ceph.yaml6
-rw-r--r--qa/cephfs/begin/2-logrotate.yaml3
l---------qa/cephfs/clusters/.qa1
-rw-r--r--qa/cephfs/clusters/1-mds-1-client-coloc.yaml9
-rw-r--r--qa/cephfs/clusters/1-mds-1-client-micro.yaml7
-rw-r--r--qa/cephfs/clusters/1-mds-1-client.yaml10
-rw-r--r--qa/cephfs/clusters/1-mds-2-client-coloc.yaml9
-rw-r--r--qa/cephfs/clusters/1-mds-2-client-micro.yaml8
-rw-r--r--qa/cephfs/clusters/1-mds-2-client.yaml11
-rw-r--r--qa/cephfs/clusters/1-mds-3-client.yaml12
-rw-r--r--qa/cephfs/clusters/1-mds-4-client-coloc.yaml9
-rw-r--r--qa/cephfs/clusters/1-mds-4-client.yaml13
-rw-r--r--qa/cephfs/clusters/1-node-1-mds-1-osd.yaml8
-rw-r--r--qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml10
-rw-r--r--qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml10
-rw-r--r--qa/cephfs/clusters/1a3s-mds-1c-client.yaml9
-rw-r--r--qa/cephfs/clusters/1a3s-mds-2c-client.yaml9
-rw-r--r--qa/cephfs/clusters/1a3s-mds-4c-client.yaml9
-rw-r--r--qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml10
-rw-r--r--qa/cephfs/clusters/1a5s-mds-1c-client.yaml9
-rw-r--r--qa/cephfs/clusters/3-mds.yaml14
-rw-r--r--qa/cephfs/clusters/9-mds.yaml14
-rw-r--r--qa/cephfs/clusters/fixed-2-ucephfs.yaml9
-rw-r--r--qa/cephfs/conf/+0
l---------qa/cephfs/conf/.qa1
-rw-r--r--qa/cephfs/conf/client.yaml9
-rw-r--r--qa/cephfs/conf/mds.yaml14
-rw-r--r--qa/cephfs/conf/mon.yaml5
-rw-r--r--qa/cephfs/conf/osd.yaml5
l---------qa/cephfs/mount/.qa1
-rw-r--r--qa/cephfs/mount/fuse.yaml16
-rw-r--r--qa/cephfs/mount/kclient/%0
l---------qa/cephfs/mount/kclient/.qa1
-rw-r--r--qa/cephfs/mount/kclient/mount-syntax/$0
-rw-r--r--qa/cephfs/mount/kclient/mount-syntax/v1.yaml3
-rw-r--r--qa/cephfs/mount/kclient/mount-syntax/v2.yaml3
-rw-r--r--qa/cephfs/mount/kclient/mount.yaml16
-rw-r--r--qa/cephfs/mount/kclient/overrides/%0
l---------qa/cephfs/mount/kclient/overrides/.qa1
l---------qa/cephfs/mount/kclient/overrides/distro/.qa1
-rw-r--r--qa/cephfs/mount/kclient/overrides/distro/stock/%0
l---------qa/cephfs/mount/kclient/overrides/distro/stock/.qa1
-rw-r--r--qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml4
l---------qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml1
l---------qa/cephfs/mount/kclient/overrides/distro/testing/.qa1
-rw-r--r--qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml4
-rw-r--r--qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml5
l---------qa/cephfs/objectstore-ec/.qa1
l---------qa/cephfs/objectstore-ec/bluestore-bitmap.yaml1
-rw-r--r--qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml29
-rw-r--r--qa/cephfs/objectstore-ec/bluestore-comp.yaml23
-rw-r--r--qa/cephfs/objectstore-ec/bluestore-ec-root.yaml43
l---------qa/cephfs/overrides/.qa1
-rw-r--r--qa/cephfs/overrides/frag.yaml9
l---------qa/cephfs/overrides/fuse/.qa1
-rw-r--r--qa/cephfs/overrides/fuse/default-perm/%0
l---------qa/cephfs/overrides/fuse/default-perm/.qa1
-rw-r--r--qa/cephfs/overrides/fuse/default-perm/no.yaml5
-rw-r--r--qa/cephfs/overrides/fuse/default-perm/yes.yaml5
-rw-r--r--qa/cephfs/overrides/ignorelist_health.yaml13
-rw-r--r--qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml9
-rw-r--r--qa/cephfs/overrides/osd-asserts.yaml5
-rw-r--r--qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml5
-rw-r--r--qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml5
-rw-r--r--qa/cephfs/overrides/session_timeout.yaml4
l---------qa/cephfs/tasks/.qa1
-rw-r--r--qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml9
-rw-r--r--qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml5
-rw-r--r--qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml17
-rw-r--r--qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml6
-rw-r--r--qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml5
-rw-r--r--qa/cephfs/tasks/libcephfs_interface_tests.yaml14
-rwxr-xr-xqa/cephfs/unshare_ns_mount.sh594
76 files changed, 1178 insertions, 0 deletions
diff --git a/qa/cephfs/.qa b/qa/cephfs/.qa
new file mode 120000
index 000000000..b870225aa
--- /dev/null
+++ b/qa/cephfs/.qa
@@ -0,0 +1 @@
+../ \ No newline at end of file
diff --git a/qa/cephfs/begin/+ b/qa/cephfs/begin/+
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/begin/+
diff --git a/qa/cephfs/begin/0-install.yaml b/qa/cephfs/begin/0-install.yaml
new file mode 100644
index 000000000..413958f11
--- /dev/null
+++ b/qa/cephfs/begin/0-install.yaml
@@ -0,0 +1,65 @@
+tasks:
+ - install:
+ extra_packages:
+ rpm:
+ - python3-cephfs
+ - cephfs-top
+ - cephfs-mirror
+ deb:
+ - python3-cephfs
+ - cephfs-shell
+ - cephfs-top
+ - cephfs-mirror
+ # For kernel_untar_build workunit
+ extra_system_packages:
+ deb:
+ - bison
+ - flex
+ - libelf-dev
+ - libssl-dev
+ - network-manager
+ - iproute2
+ - util-linux
+ # for xfstests-dev
+ - dump
+ - indent
+ # for fsx
+ - libaio-dev
+ - libtool-bin
+ - uuid-dev
+ - xfslibs-dev
+ # for postgres
+ - postgresql
+ - postgresql-client
+ - postgresql-common
+ - postgresql-contrib
+ rpm:
+ - bison
+ - flex
+ - elfutils-libelf-devel
+ - openssl-devel
+ - NetworkManager
+ - iproute
+ - util-linux
+ # for xfstests-dev
+ - libacl-devel
+ - libaio-devel
+ - libattr-devel
+ - libtool
+ - libuuid-devel
+ - xfsdump
+ - xfsprogs
+ - xfsprogs-devel
+ # for fsx
+ - libaio-devel
+ - libtool
+ - libuuid-devel
+ - xfsprogs-devel
+ # for postgres
+ - postgresql
+ - postgresql-server
+ - postgresql-contrib
+syslog:
+ ignorelist:
+ - WARNING*.*check_session_state
+ - WARNING*.*__ceph_remove_cap
diff --git a/qa/cephfs/begin/1-ceph.yaml b/qa/cephfs/begin/1-ceph.yaml
new file mode 100644
index 000000000..531c8e3e0
--- /dev/null
+++ b/qa/cephfs/begin/1-ceph.yaml
@@ -0,0 +1,6 @@
+log-rotate:
+ ceph-mds: 10G
+ ceph-osd: 10G
+tasks:
+ - ceph:
+ create_rbd_pool: false
diff --git a/qa/cephfs/begin/2-logrotate.yaml b/qa/cephfs/begin/2-logrotate.yaml
new file mode 100644
index 000000000..3b876f13b
--- /dev/null
+++ b/qa/cephfs/begin/2-logrotate.yaml
@@ -0,0 +1,3 @@
+log-rotate:
+ ceph-mds: 10G
+ ceph-osd: 10G
diff --git a/qa/cephfs/clusters/.qa b/qa/cephfs/clusters/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/clusters/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/clusters/1-mds-1-client-coloc.yaml b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml
new file mode 100644
index 000000000..d295dc3d3
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-1-client-micro.yaml b/qa/cephfs/clusters/1-mds-1-client-micro.yaml
new file mode 100644
index 000000000..8b66c3906
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-1-client-micro.yaml
@@ -0,0 +1,7 @@
+roles:
+- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3]
+- [client.0]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
diff --git a/qa/cephfs/clusters/1-mds-1-client.yaml b/qa/cephfs/clusters/1-mds-1-client.yaml
new file mode 100644
index 000000000..d7701815f
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-1-client.yaml
@@ -0,0 +1,10 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+- [client.0]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-2-client-coloc.yaml b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml
new file mode 100644
index 000000000..5ce128cbd
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.1]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-2-client-micro.yaml b/qa/cephfs/clusters/1-mds-2-client-micro.yaml
new file mode 100644
index 000000000..42ad4efcd
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-2-client-micro.yaml
@@ -0,0 +1,8 @@
+roles:
+- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.a, mds.b, mds.c, osd.0, osd.1, osd.2, osd.3]
+- [client.0]
+- [client.1]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 10 # GB
diff --git a/qa/cephfs/clusters/1-mds-2-client.yaml b/qa/cephfs/clusters/1-mds-2-client.yaml
new file mode 100644
index 000000000..6e996244e
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-2-client.yaml
@@ -0,0 +1,11 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+- [client.0]
+- [client.1]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-3-client.yaml b/qa/cephfs/clusters/1-mds-3-client.yaml
new file mode 100644
index 000000000..200df1fe0
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-3-client.yaml
@@ -0,0 +1,12 @@
+roles:
+- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+- [client.0]
+- [client.1]
+- [client.2]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-4-client-coloc.yaml b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml
new file mode 100644
index 000000000..3e2ee7870
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0, client.1]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.2, client.3]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-mds-4-client.yaml b/qa/cephfs/clusters/1-mds-4-client.yaml
new file mode 100644
index 000000000..9addfe3b0
--- /dev/null
+++ b/qa/cephfs/clusters/1-mds-4-client.yaml
@@ -0,0 +1,13 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.b, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mon.c, mgr.x, mds.c, osd.4, osd.5, osd.6, osd.7]
+- [client.0]
+- [client.1]
+- [client.2]
+- [client.3]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml b/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml
new file mode 100644
index 000000000..865b976c6
--- /dev/null
+++ b/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml
@@ -0,0 +1,8 @@
+roles:
+- [mon.a, mgr.x, mds.a, osd.0, client.0]
+openstack:
+- volumes: # attached to each instance
+ count: 1
+ size: 5 # GB
+- machine:
+ disk: 10 # GB
diff --git a/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml
new file mode 100644
index 000000000..ca6f79ba2
--- /dev/null
+++ b/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml
@@ -0,0 +1,10 @@
+roles:
+- [mon.a, mgr.x, mds.a, mds.d, mds.g, mds.j, osd.0, osd.3, osd.6, osd.9, client.0]
+- [mon.b, mgr.y, mds.b, mds.e, mds.h, mds.k, osd.1, osd.4, osd.7, osd.10]
+- [mon.c, mgr.z, mds.c, mds.f, mds.i, mds.l, osd.2, osd.5, osd.8, osd.11]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml
new file mode 100644
index 000000000..3a9ff34f8
--- /dev/null
+++ b/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml
@@ -0,0 +1,10 @@
+roles:
+- [mon.a, mgr.x, mds.a, osd.0, osd.3, osd.6, osd.9, client.0]
+- [mon.b, mgr.y, mds.b, osd.1, osd.4, osd.7, osd.10]
+- [mon.c, mgr.z, mds.c, osd.2, osd.5, osd.8, osd.11]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a3s-mds-1c-client.yaml b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml
new file mode 100644
index 000000000..87867daa4
--- /dev/null
+++ b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a3s-mds-2c-client.yaml b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml
new file mode 100644
index 000000000..ba17f05d2
--- /dev/null
+++ b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7, client.1]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a3s-mds-4c-client.yaml b/qa/cephfs/clusters/1a3s-mds-4c-client.yaml
new file mode 100644
index 000000000..51fd809b4
--- /dev/null
+++ b/qa/cephfs/clusters/1a3s-mds-4c-client.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0, client.2]
+- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7, client.1, client.3]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml
new file mode 100644
index 000000000..ae723c160
--- /dev/null
+++ b/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml
@@ -0,0 +1,10 @@
+roles:
+- [mon.a, mgr.x, mds.a, mds.d, osd.0, osd.3, osd.6, osd.9, client.0]
+- [mon.b, mgr.y, mds.b, mds.e, osd.1, osd.4, osd.7, osd.10]
+- [mon.c, mgr.z, mds.c, mds.f, osd.2, osd.5, osd.8, osd.11]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/1a5s-mds-1c-client.yaml b/qa/cephfs/clusters/1a5s-mds-1c-client.yaml
new file mode 100644
index 000000000..79fd84cf4
--- /dev/null
+++ b/qa/cephfs/clusters/1a5s-mds-1c-client.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, mds.c, mds.e, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, mds.d, mds.f, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 20 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/3-mds.yaml b/qa/cephfs/clusters/3-mds.yaml
new file mode 100644
index 000000000..8dfacb4e7
--- /dev/null
+++ b/qa/cephfs/clusters/3-mds.yaml
@@ -0,0 +1,14 @@
+roles:
+- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7]
+- [client.0, client.1]
+overrides:
+ ceph:
+ cephfs:
+ max_mds: 3
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/9-mds.yaml b/qa/cephfs/clusters/9-mds.yaml
new file mode 100644
index 000000000..60b18c251
--- /dev/null
+++ b/qa/cephfs/clusters/9-mds.yaml
@@ -0,0 +1,14 @@
+roles:
+- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3]
+- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7]
+- [client.0, client.1]
+overrides:
+ ceph:
+ cephfs:
+ max_mds: 9
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/clusters/fixed-2-ucephfs.yaml b/qa/cephfs/clusters/fixed-2-ucephfs.yaml
new file mode 100644
index 000000000..dff37a51f
--- /dev/null
+++ b/qa/cephfs/clusters/fixed-2-ucephfs.yaml
@@ -0,0 +1,9 @@
+roles:
+- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7]
+openstack:
+- volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
+- machine:
+ disk: 200 # GB
diff --git a/qa/cephfs/conf/+ b/qa/cephfs/conf/+
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/conf/+
diff --git a/qa/cephfs/conf/.qa b/qa/cephfs/conf/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/conf/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/conf/client.yaml b/qa/cephfs/conf/client.yaml
new file mode 100644
index 000000000..4b37d03b4
--- /dev/null
+++ b/qa/cephfs/conf/client.yaml
@@ -0,0 +1,9 @@
+overrides:
+ ceph:
+ conf:
+ client:
+ client mount timeout: 600
+ debug ms: 1
+ debug client: 20
+ rados mon op timeout: 900
+ rados osd op timeout: 900
diff --git a/qa/cephfs/conf/mds.yaml b/qa/cephfs/conf/mds.yaml
new file mode 100644
index 000000000..b1c7a5869
--- /dev/null
+++ b/qa/cephfs/conf/mds.yaml
@@ -0,0 +1,14 @@
+overrides:
+ ceph:
+ conf:
+ mds:
+ debug mds: 20
+ debug mds balancer: 20
+ debug ms: 1
+ mds debug frag: true
+ mds debug scatterstat: true
+ mds op complaint time: 180
+ mds verify scatter: true
+ osd op complaint time: 180
+ rados mon op timeout: 900
+ rados osd op timeout: 900
diff --git a/qa/cephfs/conf/mon.yaml b/qa/cephfs/conf/mon.yaml
new file mode 100644
index 000000000..eea56004a
--- /dev/null
+++ b/qa/cephfs/conf/mon.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mon:
+ mon op complaint time: 120
diff --git a/qa/cephfs/conf/osd.yaml b/qa/cephfs/conf/osd.yaml
new file mode 100644
index 000000000..1087202f9
--- /dev/null
+++ b/qa/cephfs/conf/osd.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd op complaint time: 180
diff --git a/qa/cephfs/mount/.qa b/qa/cephfs/mount/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/fuse.yaml b/qa/cephfs/mount/fuse.yaml
new file mode 100644
index 000000000..9ffb5129d
--- /dev/null
+++ b/qa/cephfs/mount/fuse.yaml
@@ -0,0 +1,16 @@
+teuthology:
+ postmerge:
+ - local function is_kupstream()
+ return false
+ end
+ - local function is_kdistro()
+ return false
+ end
+ - local function is_fuse()
+ return true
+ end
+ - local function syntax_version()
+ return ''
+ end
+tasks:
+ - ceph-fuse:
diff --git a/qa/cephfs/mount/kclient/% b/qa/cephfs/mount/kclient/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/mount/kclient/%
diff --git a/qa/cephfs/mount/kclient/.qa b/qa/cephfs/mount/kclient/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/kclient/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/mount-syntax/$ b/qa/cephfs/mount/kclient/mount-syntax/$
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/mount/kclient/mount-syntax/$
diff --git a/qa/cephfs/mount/kclient/mount-syntax/v1.yaml b/qa/cephfs/mount/kclient/mount-syntax/v1.yaml
new file mode 100644
index 000000000..84d5d43b2
--- /dev/null
+++ b/qa/cephfs/mount/kclient/mount-syntax/v1.yaml
@@ -0,0 +1,3 @@
+overrides:
+ kclient:
+ syntax: 'v1'
diff --git a/qa/cephfs/mount/kclient/mount-syntax/v2.yaml b/qa/cephfs/mount/kclient/mount-syntax/v2.yaml
new file mode 100644
index 000000000..ef7d30424
--- /dev/null
+++ b/qa/cephfs/mount/kclient/mount-syntax/v2.yaml
@@ -0,0 +1,3 @@
+overrides:
+ kclient:
+ syntax: 'v2'
diff --git a/qa/cephfs/mount/kclient/mount.yaml b/qa/cephfs/mount/kclient/mount.yaml
new file mode 100644
index 000000000..a130ff9d5
--- /dev/null
+++ b/qa/cephfs/mount/kclient/mount.yaml
@@ -0,0 +1,16 @@
+teuthology:
+ postmerge:
+ - local function is_kupstream()
+ return yaml.ktype == 'upstream'
+ end
+ - local function is_kdistro()
+ return yaml.ktype == 'distro'
+ end
+ - local function is_fuse()
+ return false
+ end
+ - local function syntax_version()
+ return yaml.overrides.kclient.syntax
+ end
+tasks:
+- kclient:
diff --git a/qa/cephfs/mount/kclient/overrides/% b/qa/cephfs/mount/kclient/overrides/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/%
diff --git a/qa/cephfs/mount/kclient/overrides/.qa b/qa/cephfs/mount/kclient/overrides/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/.qa b/qa/cephfs/mount/kclient/overrides/distro/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/% b/qa/cephfs/mount/kclient/overrides/distro/stock/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/stock/%
diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/.qa b/qa/cephfs/mount/kclient/overrides/distro/stock/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/stock/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml
new file mode 100644
index 000000000..ca2d688bb
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml
@@ -0,0 +1,4 @@
+kernel:
+ client:
+ sha1: distro
+ktype: distro
diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml
new file mode 120000
index 000000000..133acf27b
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml
@@ -0,0 +1 @@
+.qa/distros/all/rhel_8.yaml \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/testing/.qa b/qa/cephfs/mount/kclient/overrides/distro/testing/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/testing/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml b/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml
new file mode 100644
index 000000000..2ee219125
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml
@@ -0,0 +1,4 @@
+kernel:
+ client:
+ branch: testing
+ktype: upstream
diff --git a/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml b/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml
new file mode 100644
index 000000000..30da870b2
--- /dev/null
+++ b/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ global:
+ ms die on skipped message: false
diff --git a/qa/cephfs/objectstore-ec/.qa b/qa/cephfs/objectstore-ec/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/objectstore-ec/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml b/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml
new file mode 120000
index 000000000..9fb86b9fe
--- /dev/null
+++ b/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+../../objectstore/bluestore-bitmap.yaml \ No newline at end of file
diff --git a/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml b/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml
new file mode 100644
index 000000000..512eb117c
--- /dev/null
+++ b/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml
@@ -0,0 +1,29 @@
+overrides:
+ thrashosds:
+ bdev_inject_crash: 2
+ bdev_inject_crash_probability: .5
+ ceph:
+ fs: xfs
+ cephfs:
+ ec_profile:
+ - m=2
+ - k=2
+ - crush-failure-domain=osd
+ conf:
+ osd:
+ osd objectstore: bluestore
+ bluestore block size: 96636764160
+ debug bluestore: 20
+ debug bluefs: 20
+ debug rocksdb: 10
+ bluestore compression mode: aggressive
+ bluestore fsck on mount: true
+ # lower the full ratios since we can fill up a 100gb osd so quickly
+ mon osd full ratio: .9
+ mon osd backfillfull_ratio: .85
+ mon osd nearfull ratio: .8
+ osd failsafe full ratio: .95
+
+# this doesn't work with failures bc the log writes are not atomic across the two backends
+# bluestore bluefs env mirror: true
+
diff --git a/qa/cephfs/objectstore-ec/bluestore-comp.yaml b/qa/cephfs/objectstore-ec/bluestore-comp.yaml
new file mode 100644
index 000000000..b408032fd
--- /dev/null
+++ b/qa/cephfs/objectstore-ec/bluestore-comp.yaml
@@ -0,0 +1,23 @@
+overrides:
+ thrashosds:
+ bdev_inject_crash: 2
+ bdev_inject_crash_probability: .5
+ ceph:
+ fs: xfs
+ conf:
+ osd:
+ osd objectstore: bluestore
+ bluestore block size: 96636764160
+ debug bluestore: 20
+ debug bluefs: 20
+ debug rocksdb: 10
+ bluestore compression mode: aggressive
+ bluestore fsck on mount: true
+ # lower the full ratios since we can fill up a 100gb osd so quickly
+ mon osd full ratio: .9
+ mon osd backfillfull_ratio: .85
+ mon osd nearfull ratio: .8
+ osd failsafe full ratio: .95
+
+# this doesn't work with failures bc the log writes are not atomic across the two backends
+# bluestore bluefs env mirror: true
diff --git a/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml b/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml
new file mode 100644
index 000000000..b89c4c711
--- /dev/null
+++ b/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml
@@ -0,0 +1,43 @@
+overrides:
+ thrashosds:
+ bdev_inject_crash: 2
+ bdev_inject_crash_probability: .5
+ ceph:
+ fs: xfs
+ cephfs:
+ ec_profile:
+ - m=2
+ - k=2
+ - crush-failure-domain=osd
+ conf:
+ osd:
+ osd objectstore: bluestore
+ bluestore block size: 96636764160
+ debug bluestore: 20
+ debug bluefs: 20
+ debug rocksdb: 10
+ bluestore fsck on mount: true
+ # lower the full ratios since we can fill up a 100gb osd so quickly
+ mon osd full ratio: .9
+ mon osd backfillfull_ratio: .85
+ mon osd nearfull ratio: .8
+ osd failsafe full ratio: .95
+# this doesn't work with failures bc the log writes are not atomic across the two backends
+# bluestore bluefs env mirror: true
+ ceph-deploy:
+ fs: xfs
+ bluestore: yes
+ conf:
+ osd:
+ osd objectstore: bluestore
+ bluestore block size: 96636764160
+ debug bluestore: 20
+ debug bluefs: 20
+ debug rocksdb: 10
+ bluestore fsck on mount: true
+ # lower the full ratios since we can fill up a 100gb osd so quickly
+ mon osd full ratio: .9
+ mon osd backfillfull_ratio: .85
+ mon osd nearfull ratio: .8
+ osd failsafe full ratio: .95
+
diff --git a/qa/cephfs/overrides/.qa b/qa/cephfs/overrides/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/overrides/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/overrides/frag.yaml b/qa/cephfs/overrides/frag.yaml
new file mode 100644
index 000000000..f05b3f48f
--- /dev/null
+++ b/qa/cephfs/overrides/frag.yaml
@@ -0,0 +1,9 @@
+overrides:
+ ceph:
+ conf:
+ mds:
+ mds bal fragment size max: 10000
+ mds bal merge size: 5
+ mds bal split bits: 3
+ mds bal split size: 100
+
diff --git a/qa/cephfs/overrides/fuse/.qa b/qa/cephfs/overrides/fuse/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/overrides/fuse/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/overrides/fuse/default-perm/% b/qa/cephfs/overrides/fuse/default-perm/%
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qa/cephfs/overrides/fuse/default-perm/%
diff --git a/qa/cephfs/overrides/fuse/default-perm/.qa b/qa/cephfs/overrides/fuse/default-perm/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/overrides/fuse/default-perm/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/overrides/fuse/default-perm/no.yaml b/qa/cephfs/overrides/fuse/default-perm/no.yaml
new file mode 100644
index 000000000..445e93652
--- /dev/null
+++ b/qa/cephfs/overrides/fuse/default-perm/no.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ client:
+ fuse default permissions: false
diff --git a/qa/cephfs/overrides/fuse/default-perm/yes.yaml b/qa/cephfs/overrides/fuse/default-perm/yes.yaml
new file mode 100644
index 000000000..2fd210a39
--- /dev/null
+++ b/qa/cephfs/overrides/fuse/default-perm/yes.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ client:
+ fuse default permissions: true
diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml
new file mode 100644
index 000000000..d8b819288
--- /dev/null
+++ b/qa/cephfs/overrides/ignorelist_health.yaml
@@ -0,0 +1,13 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - overall HEALTH_
+ - \(FS_DEGRADED\)
+ - \(MDS_FAILED\)
+ - \(MDS_DEGRADED\)
+ - \(FS_WITH_FAILED_MDS\)
+ - \(MDS_DAMAGE\)
+ - \(MDS_ALL_DOWN\)
+ - \(MDS_UP_LESS_THAN_MAX\)
+ - \(FS_INLINE_DATA_DEPRECATED\)
+ - \(POOL_APP_NOT_ENABLED\)
diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
new file mode 100644
index 000000000..41ba84f04
--- /dev/null
+++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml
@@ -0,0 +1,9 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - overall HEALTH_
+ - \(OSD_DOWN\)
+ - \(OSD_
+ - but it is still running
+# MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a'
+ - is not responding
diff --git a/qa/cephfs/overrides/osd-asserts.yaml b/qa/cephfs/overrides/osd-asserts.yaml
new file mode 100644
index 000000000..8c16e6e1c
--- /dev/null
+++ b/qa/cephfs/overrides/osd-asserts.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ osd:
+ osd shutdown pgref assert: true
diff --git a/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml b/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml
new file mode 100644
index 000000000..71d6d73ba
--- /dev/null
+++ b/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mds:
+ mds_dir_prefetch: false
diff --git a/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml b/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml
new file mode 100644
index 000000000..5d25b61d1
--- /dev/null
+++ b/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml
@@ -0,0 +1,5 @@
+overrides:
+ ceph:
+ conf:
+ mds:
+ mds_dir_prefetch: true
diff --git a/qa/cephfs/overrides/session_timeout.yaml b/qa/cephfs/overrides/session_timeout.yaml
new file mode 100644
index 000000000..a7a163337
--- /dev/null
+++ b/qa/cephfs/overrides/session_timeout.yaml
@@ -0,0 +1,4 @@
+overrides:
+ ceph:
+ cephfs:
+ session_timeout: 300
diff --git a/qa/cephfs/tasks/.qa b/qa/cephfs/tasks/.qa
new file mode 120000
index 000000000..a602a0353
--- /dev/null
+++ b/qa/cephfs/tasks/.qa
@@ -0,0 +1 @@
+../.qa/ \ No newline at end of file
diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml
new file mode 100644
index 000000000..2d370d7ef
--- /dev/null
+++ b/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml
@@ -0,0 +1,9 @@
+tasks:
+- check-counter:
+ counters:
+ mds:
+ - "mds.dir_split"
+- workunit:
+ clients:
+ all:
+ - suites/blogbench.sh
diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml
new file mode 100644
index 000000000..41b2bc8ed
--- /dev/null
+++ b/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml
@@ -0,0 +1,5 @@
+tasks:
+- workunit:
+ clients:
+ all:
+ - suites/dbench.sh
diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml
new file mode 100644
index 000000000..6a2b35a18
--- /dev/null
+++ b/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml
@@ -0,0 +1,17 @@
+overrides:
+ ceph:
+ log-ignorelist:
+ - SLOW_OPS
+ - slow request
+ conf:
+ osd:
+ filestore flush min: 0
+tasks:
+- check-counter:
+ counters:
+ mds:
+ - "mds.dir_split"
+- workunit:
+ clients:
+ all:
+ - suites/ffsb.sh
diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml
new file mode 100644
index 000000000..bae220292
--- /dev/null
+++ b/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml
@@ -0,0 +1,6 @@
+tasks:
+- workunit:
+ timeout: 6h
+ clients:
+ all:
+ - suites/fsstress.sh
diff --git a/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml b/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml
new file mode 100644
index 000000000..e51542022
--- /dev/null
+++ b/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml
@@ -0,0 +1,5 @@
+tasks:
+- workunit:
+ clients:
+ all:
+ - fs/misc/trivial_sync.sh
diff --git a/qa/cephfs/tasks/libcephfs_interface_tests.yaml b/qa/cephfs/tasks/libcephfs_interface_tests.yaml
new file mode 100644
index 000000000..c59775259
--- /dev/null
+++ b/qa/cephfs/tasks/libcephfs_interface_tests.yaml
@@ -0,0 +1,14 @@
+overrides:
+ ceph-fuse:
+ disabled: true
+ kclient:
+ disabled: true
+tasks:
+- check-counter:
+ counters:
+ mds:
+ - "mds.dir_split"
+- workunit:
+ clients:
+ client.0:
+ - libcephfs/test.sh
diff --git a/qa/cephfs/unshare_ns_mount.sh b/qa/cephfs/unshare_ns_mount.sh
new file mode 100755
index 000000000..88ac3e933
--- /dev/null
+++ b/qa/cephfs/unshare_ns_mount.sh
@@ -0,0 +1,594 @@
+#!/usr/bin/env bash
+
+# This is one helper for mounting the ceph-fuse/kernel clients by
+# unsharing the network namespace, let's call it netns container.
+# With the netns container, you can easily suspend or resume the
+# virtual network interface to simulate the client node hard
+# shutdown for some test cases.
+#
+# netnsX netnsY netnsZ
+# -------------- -------------- --------------
+# | mount client | | mount client | | mount client |
+# | default | ... | default | ... | default |
+# |192.168.0.1/16| |192.168.0.2/16| |192.168.0.3/16|
+# | veth0 | | veth0 | | veth0 |
+# -------------- -------------- -------------
+# | | |
+# \ | brx.Y /
+# \ ---------------------- /
+# \ brx.X | ceph-brx | brx.Z /
+# \------>| default |<------/
+# | | 192.168.255.254/16 | |
+# | ---------------------- |
+# (suspend/resume) | (suspend/resume)
+# -----------
+# | Physical |
+# | A.B.C.D/M |
+# -----------
+#
+# Defaultly it will use the 192.168.X.Y/16 private network IPs for
+# the ceph-brx and netnses as above. And you can also specify your
+# own new ip/mask for the ceph-brx, like:
+#
+# $ unshare_ns_mount.sh --fuse /mnt/cephfs --brxip 172.19.100.100/12
+#
+# Then the each netns will get a new ip from the ranges:
+# [172.16.0.1 ~ 172.19.100.99]/12 and [172.19.100.101 ~ 172.31.255.254]/12
+
+usage() {
+ echo ""
+ echo "This will help to isolate the network namespace from OS for the mount client!"
+ echo ""
+ echo "usage: unshare_ns_mount.sh [OPTIONS [paramters]] [--brxip <ip_address/mask>]"
+ echo "OPTIONS:"
+ echo -e " --fuse <ceph-fuse options>"
+ echo -e "\tThe ceph-fuse command options"
+ echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs -o nonempty"
+ echo ""
+ echo -e " --kernel <mount options>"
+ echo -e "\tThe mount command options"
+ echo -e "\t $ unshare_ns_mount.sh --kernel -t ceph 192.168.0.1:6789:/ /mnt/cephfs -o fs=a"
+ echo ""
+ echo -e " --suspend <mountpoint>"
+ echo -e "\tDown the veth interface in the network namespace"
+ echo -e "\t $ unshare_ns_mount.sh --suspend /mnt/cephfs"
+ echo ""
+ echo -e " --resume <mountpoint>"
+ echo -e "\tUp the veth interface in the network namespace"
+ echo -e "\t $ unshare_ns_mount.sh --resume /mnt/cephfs"
+ echo ""
+ echo -e " --umount <mountpoint>"
+ echo -e "\tUmount and delete the network namespace"
+ echo -e "\t $ unshare_ns_mount.sh --umount /mnt/cephfs"
+ echo ""
+ echo -e " --brxip <ip_address/mask>"
+ echo -e "\tSpecify ip/mask for ceph-brx and it only makes sense for --fuse/--kernel options"
+ echo -e "\t(default: 192.168.255.254/16, netns ip: 192.168.0.1/16 ~ 192.168.255.253/16)"
+ echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs --brxip 172.19.255.254/12"
+ echo -e "\t $ unshare_ns_mount.sh --kernel 192.168.0.1:6789:/ /mnt/cephfs --brxip 172.19.255.254/12"
+ echo ""
+ echo -e " -h, --help"
+ echo -e "\tPrint help"
+ echo ""
+}
+
+CEPH_BRX=ceph-brx
+CEPH_BRX_IP_DEF=192.168.255.254
+NET_MASK_DEF=16
+BRD_DEF=192.168.255.255
+
+CEPH_BRX_IP=$CEPH_BRX_IP_DEF
+NET_MASK=$NET_MASK_DEF
+BRD=$BRD_DEF
+
+mountpoint=""
+new_netns=""
+fuse_type=false
+
+function get_mountpoint() {
+ for param in $@
+ do
+ if [ -d $param ]; then
+ # skipping "--client_mountpoint/-r root_directory"
+ # option for ceph-fuse command
+ if [ "$last" == "-r" -o "$last" == "--client_mountpoint" ]; then
+ last=$param
+ continue
+ fi
+ if [ "0$mountpoint" != "0" ]; then
+ echo "Oops: too many mountpiont options!"
+ exit 1
+ fi
+ mountpoint=$param
+ fi
+ last=$param
+ done
+
+ if [ "0$mountpoint" == "0" ]; then
+ echo "Oops: mountpoint path is not a directory or no mountpoint specified!"
+ exit 1
+ fi
+}
+
+function get_new_netns() {
+ # prune the repeating slashes:
+ # "/mnt///cephfs///" --> "/mnt/cephfs/"
+ __mountpoint=`echo "$mountpoint" | sed 's/\/\+/\//g'`
+
+ # prune the leading slashes
+ while [ ${__mountpoint:0:1} == "/" ]
+ do
+ __mountpoint=${__mountpoint:1}
+ done
+
+ # prune the last slashes
+ while [ ${__mountpoint: -1} == "/" ]
+ do
+ __mountpoint=${__mountpoint:0:-1}
+ done
+
+ # replace '/' with '-'
+ __mountpoint=${__mountpoint//\//-}
+
+ # "mnt/cephfs" --> "ceph-fuse-mnt-cephfs"
+ if [ "$1" == "--fuse" ]; then
+ new_netns=`echo ceph-fuse-$__mountpoint`
+ fuse_type=true
+ return
+ fi
+
+ # "mnt/cephfs" --> "ceph-kernel-mnt-cephfs"
+ if [ "$1" == "--kernel" ]; then
+ new_netns=`echo ceph-kernel-$__mountpoint`
+ return
+ fi
+
+ # we are in umount/suspend/resume routines
+ for ns in `ip netns list | awk '{print $1}'`
+ do
+ if [ "$ns" == "ceph-fuse-$__mountpoint" ]; then
+ new_netns=$ns
+ fuse_type=true
+ return
+ fi
+ if [ "$ns" == "ceph-kernel-$__mountpoint" ]; then
+ new_netns=$ns
+ return
+ fi
+ done
+
+ if [ "0$new_netns" == "0" ]; then
+ echo "Oops, netns 'ceph-{fuse/kernel}-$__mountpoint' does not exists!"
+ exit 1
+ fi
+}
+
+# the peer veth name will be "brx.$nsid" on host node
+function get_netns_brx() {
+ get_new_netns
+
+ nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'`
+ netns_veth=brx.$nsid
+ eval $1="$netns_veth"
+}
+
+function suspend_netns_veth() {
+ get_mountpoint $@
+
+ get_netns_brx brx
+ ip link set $brx down
+ exit 0
+}
+
+function resume_netns_veth() {
+ get_mountpoint $@
+
+ get_netns_brx brx
+ ip link set $brx up
+ exit 0
+}
+
+# help and usage
+if [ $# == 0 -o "$1" == "-h" -o "$1" == "--help" ]; then
+ usage
+ exit 0
+fi
+
+# suspend the veth from network namespace
+if [ $1 == "--suspend" ]; then
+ suspend_netns_veth $@
+ exit 0
+fi
+
+# resume the veth from network namespace
+if [ $1 == "--resume" ]; then
+ resume_netns_veth $@
+ exit 0
+fi
+
+function ceph_umount() {
+ get_mountpoint $@
+ get_new_netns
+
+ if [ $fuse_type == true ]; then
+ nsenter --net=/var/run/netns/$new_netns fusermount -u $mountpoint 2>/dev/null
+ else
+ nsenter --net=/var/run/netns/$new_netns umount $mountpoint 2>/dev/null
+ fi
+
+ # let's wait for a while to let the umount operation
+ # to finish before deleting the netns
+ while [ 1 ]
+ do
+ for pid in `ip netns pids $new_netns 2>/dev/null`
+ do
+ name=`cat /proc/$pid/comm 2>/dev/null`
+ if [ "$name" == "ceph-fuse" ]; then
+ break
+ fi
+ done
+
+ if [ "$name" == "ceph-fuse" ]; then
+ name=""
+ usleep 100000
+ continue
+ fi
+
+ break
+ done
+
+ nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'`
+ netns_brx=brx.$nsid
+
+ # brctl delif $CEPH_BRX $netns_brx 2>/dev/null
+ nmcli connection down $netns_brx down 2>/dev/null
+ nmcli connection delete $netns_brx 2>/dev/null
+
+ ip netns delete $new_netns 2>/dev/null
+
+ # if this is the last netns_brx, will delete
+ # the $CEPH_BRX and restore the OS configure
+ # rc=`brctl show ceph-brx 2>/dev/null | grep 'brx\.'|wc -l`
+ rc=`nmcli connection show 2>/dev/null | grep 'brx\.' | wc -l`
+ if [ $rc == 0 ]; then
+ ip link set $CEPH_BRX down 2>/dev/null
+ # brctl delbr $CEPH_BRX 2>/dev/null
+ nmcli connection delete $CEPH_BRX 2>/dev/null
+
+ # restore the ip forward
+ tmpfile=`ls /tmp/ | grep "$CEPH_BRX\."`
+ tmpfile=/tmp/$tmpfile
+ if [ ! -f $tmpfile ]; then
+ echo "Oops, the $CEPH_BRX.XXX temp file does not exist!"
+ else
+ save=`cat $tmpfile`
+ echo $save > /proc/sys/net/ipv4/ip_forward
+ rm -rf $tmpfile
+ fi
+
+ # drop the iptables NAT rules
+ host_nic=`route | grep default | awk '{print $8}'`
+ iptables -D FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT
+ iptables -D FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT
+ iptables -t nat -D POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE
+ fi
+}
+
+function get_brd_mask() {
+ first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'`
+ second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'`
+ third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'`
+ fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'`
+
+ if [ "$first" == "172" ]; then
+ second_max=31
+ else
+ second_max=255
+ fi
+ third_max=255
+ fourth_max=255
+
+ if [ $NET_MASK -lt 16 ]; then
+ let power=16-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ second=$((second&~m))
+ let second_max=$second+$m
+ elif [ $NET_MASK -lt 24 ]; then
+ let power=24-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ third=$((third&~m))
+ let third_max=$third+$m
+ second_max=$second
+ elif [ $NET_MASK -lt 32 ]; then
+ let power=32-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ fourth=$((fourth&~m))
+ let fourth_max=$fourth+$m
+ second_max=$second
+ third_max=$third
+ fi
+
+ BRD=$first.$second_max.$third_max.$fourth_max
+}
+
+# As default:
+# The netns IP will be 192.168.0.1 ~ 192.168.255.253,
+# and 192.168.255.254 is saved for $CEPH_BRX
+function get_new_ns_ip() {
+ first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'`
+ second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'`
+ third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'`
+ fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'`
+
+ if [ "$first" == ""172 ]; then
+ second_max=31
+ else
+ second_max=255
+ fi
+ third_max=255
+ fourth_max=254
+
+ if [ $NET_MASK -lt 16 ]; then
+ let power=16-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ second=$((second&~m))
+ let second_max=$second+$m
+ third=0
+ fourth=1
+ elif [ $NET_MASK -lt 24 ]; then
+ let power=24-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ third=$((third&~m))
+ let third_max=$third+$m
+ second_max=$second
+ fourth=1
+ elif [ $NET_MASK -lt 32 ]; then
+ let power=32-$NET_MASK
+ m=`awk 'BEGIN{printf 2^"'$power'"-1}'`
+ fourth=$((fourth&~m))
+ let fourth+=1
+ let fourth_max=$fourth+$m-1
+ second_max=$second
+ third_max=$third
+ fi
+
+ while [ $second -le $second_max -a $third -le $third_max -a $fourth -le $fourth_max ]
+ do
+ conflict=false
+
+ # check from the existing network namespaces
+ for netns in `ip netns list | awk '{print $1}'`
+ do
+ ip=`ip netns exec $netns ip addr | grep "inet " | grep "veth0"`
+ ip=`echo "$ip" | awk '{print $2}' | awk -F/ '{print $1}'`
+ if [ "0$ip" == "0" ]; then
+ continue
+ fi
+ if [ "$first.$second.$third.$fourth" == "$ip" ]; then
+ conflict=true
+
+ let fourth+=1
+ if [ $fourth -le $fourth_max ]; then
+ break
+ fi
+
+ fourth=0
+ let third+=1
+ if [ $third -le $third_max ]; then
+ break
+ fi
+
+ third=0
+ let second+=1
+ if [ $second -le $second_max ]; then
+ break
+ fi
+
+ echo "Oops: we have ran out of the ip addresses!"
+ exit 1
+ fi
+ done
+
+ # have we found one ?
+ if [ $conflict == false ]; then
+ break
+ fi
+ done
+
+ ip=$first.$second.$third.$fourth
+ max=$first.$second_max.$third_max.$fourth_max
+ if [ "$ip" == "$max" ]; then
+ echo "Oops: we have ran out of the ip addresses!"
+ exit 1
+ fi
+
+ eval $1="$ip"
+}
+
+function check_valid_private_ip() {
+ first=`echo "$1" | awk -F. '{print $1}'`
+ second=`echo "$1" | awk -F. '{print $2}'`
+
+ # private network class A 10.0.0.0 - 10.255.255.255
+ if [ "$first" == "10" -a $NET_MASK -ge 8 ]; then
+ return
+ fi
+
+ # private network class B 172.16.0.0 - 172.31.255.255
+ if [ "$first" == "172" -a $second -ge 16 -a $second -le 31 -a $NET_MASK -ge 12 ]; then
+ return
+ fi
+
+ # private network class C 192.168.0.0 - 192.168.255.255
+ if [ "$first" == "192" -a "$second" == "168" -a $NET_MASK -ge 16 ]; then
+ return
+ fi
+
+ echo "Oops: invalid private ip address '$CEPH_BRX_IP/$NET_MASK'!"
+ exit 1
+}
+
+function setup_bridge_and_nat() {
+ # check and parse the --brxip parameter
+ is_brxip=false
+ for ip in $@
+ do
+ if [ "$ip" == "--brxip" ]; then
+ is_brxip=true
+ continue
+ fi
+ if [ $is_brxip == true ]; then
+ new_brxip=$ip
+ break
+ fi
+ done
+
+ # if the $CEPH_BRX already exists, then check the new
+ # brxip, if not match fail it without doing anything.
+ rc=`ip addr | grep "inet " | grep " $CEPH_BRX"`
+ if [ "0$rc" != "0" ]; then
+ existing_brxip=`echo "$rc" | awk '{print $2}'`
+ if [ "0$new_brxip" != "0" -a "$existing_brxip" != "$new_brxip" ]; then
+ echo "Oops: conflict with the existing $CEPH_BRX ip '$existing_brxip', new '$new_brxip'!"
+ exit 1
+ fi
+
+ CEPH_BRX_IP=`echo "$existing_brxip" | awk -F/ '{print $1}'`
+ NET_MASK=`echo "$existing_brxip" | awk -F/ '{print $2}'`
+ get_brd_mask
+ return
+ fi
+
+ # if it is the first time to run the the script or there
+ # is no any network namespace exists, we need to setup
+ # the $CEPH_BRX, if no --brxip is specified will use the
+ # default $CEPH_BRX_IP/$NET_MASK
+ if [ "0$new_brxip" != "0" ]; then
+ CEPH_BRX_IP=`echo "$new_brxip" | awk -F/ '{print $1}'`
+ NET_MASK=`echo "$new_brxip" | awk -F/ '{print $2}'`
+ get_brd_mask
+ check_valid_private_ip $CEPH_BRX_IP
+ fi
+
+ # brctl addbr $CEPH_BRX
+ nmcli connection add type bridge con-name $CEPH_BRX ifname $CEPH_BRX stp no
+ # ip link set $CEPH_BRX up
+ # ip addr add $CEPH_BRX_IP/$NET_MASK brd $BRD dev $CEPH_BRX
+ nmcli connection modify $CEPH_BRX ipv4.addresses $CEPH_BRX_IP/$NET_MASK ipv4.method manual
+ nmcli connection up $CEPH_BRX
+
+ # setup the NAT
+ rm -rf /tmp/ceph-brx.*
+ tmpfile=$(mktemp /tmp/ceph-brx.XXXXXXXX)
+ save=`cat /proc/sys/net/ipv4/ip_forward`
+ echo $save > $tmpfile
+ echo 1 > /proc/sys/net/ipv4/ip_forward
+
+ host_nic=`route | grep default | awk '{print $8}'`
+ iptables -A FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT
+ iptables -A FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT
+ iptables -t nat -A POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE
+}
+
+function __ceph_mount() {
+ # for some options like the '-t' in mount command
+ # the nsenter command will take over it, so it is
+ # hard to pass it direct to the netns.
+ # here we will create one temp file with x mode
+ tmpfile=$(mktemp /tmp/ceph-nsenter.XXXXXXXX)
+ chmod +x $tmpfile
+ if [ "$1" == "--kernel" ]; then
+ cmd=`echo "$@" | sed 's/--kernel/mount/'`
+ else
+ cmd=`echo "$@" | sed 's/--fuse/ceph-fuse/'`
+ fi
+
+ # remove the --brxip parameter
+ cmd=`echo "$cmd" | sed 's/--brxip.*\/[0-9]* //'`
+
+ # enter $new_netns and run ceph fuse client mount,
+ # we couldn't use 'ip netns exec' here because it
+ # will unshare the mount namespace.
+ echo "$cmd" > $tmpfile
+ nsenter --net=/var/run/netns/$new_netns /bin/bash $tmpfile ; echo $? > $tmpfile
+ rc=`cat $tmpfile`
+ rm -f $tmpfile
+
+ # fall back
+ if [ $rc != 0 ]; then
+ m=$mountpoint
+ mountpoint=""
+ ceph_umount $m
+ fi
+}
+
+function get_new_nsid() {
+ # get one uniq netns id
+ uniq_id=0
+ while [ 1 ]
+ do
+ rc=`ip netns list-id | grep "nsid $uniq_id "`
+ if [ "0$rc" == "0" ]; then
+ break
+ fi
+ let uniq_id+=1
+ done
+
+ eval $1="$uniq_id"
+}
+
+function ceph_mount() {
+ get_mountpoint $@
+ setup_bridge_and_nat $@
+
+ get_new_netns $1
+ rc=`ip netns list | grep "$new_netns" | awk '{print $1}'`
+ if [ "0$rc" != "0" ]; then
+ echo "Oops: the netns "$new_netns" already exists!"
+ exit 1
+ fi
+
+ get_new_nsid new_nsid
+
+ # create a new network namespace
+ ip netns add $new_netns
+ ip netns set $new_netns $new_nsid
+
+ get_new_ns_ip ns_ip
+ if [ 0"$ns_ip" == "0" ]; then
+ echo "Oops: there is no ip address could be used any more!"
+ exit 1
+ fi
+
+ # veth interface in netns
+ ns_veth=veth0
+ netns_brx=brx.$new_nsid
+
+ # setup veth interfaces
+ ip link add $ns_veth netns $new_netns type veth peer name $netns_brx
+ ip netns exec $new_netns ip addr add $ns_ip/$NET_MASK brd $BRD dev $ns_veth
+ ip netns exec $new_netns ip link set $ns_veth up
+ ip netns exec $new_netns ip link set lo up
+ ip netns exec $new_netns ip route add default via $CEPH_BRX_IP
+
+ # bring up the bridge interface and join it to $CEPH_BRX
+ # brctl addif $CEPH_BRX $netns_brx
+ nmcli connection add type bridge-slave con-name $netns_brx ifname $netns_brx master $CEPH_BRX
+ nmcli connection up $netns_brx
+ # ip link set $netns_brx up
+
+ __ceph_mount $@
+}
+
+if [ "$1" == "--umount" ]; then
+ ceph_umount $@
+ exit 0
+fi
+
+# mount in the netns
+if [ "$1" != "--kernel" -a "$1" != "--fuse" ]; then
+ echo "Oops: invalid mount options '$1'!"
+ exit 1
+fi
+
+ceph_mount $@