diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
| commit | e6918187568dbd01842d8d1d2c808ce16a894239 (patch) | |
| tree | 64f88b554b444a49f656b6c656111a145cbbaa28 /qa/cephfs | |
| parent | Initial commit. (diff) | |
| download | ceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip | |
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'qa/cephfs')
76 files changed, 1178 insertions, 0 deletions
diff --git a/qa/cephfs/.qa b/qa/cephfs/.qa new file mode 120000 index 000000000..b870225aa --- /dev/null +++ b/qa/cephfs/.qa @@ -0,0 +1 @@ +../
\ No newline at end of file diff --git a/qa/cephfs/begin/+ b/qa/cephfs/begin/+ new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/begin/+ diff --git a/qa/cephfs/begin/0-install.yaml b/qa/cephfs/begin/0-install.yaml new file mode 100644 index 000000000..413958f11 --- /dev/null +++ b/qa/cephfs/begin/0-install.yaml @@ -0,0 +1,65 @@ +tasks: + - install: + extra_packages: + rpm: + - python3-cephfs + - cephfs-top + - cephfs-mirror + deb: + - python3-cephfs + - cephfs-shell + - cephfs-top + - cephfs-mirror + # For kernel_untar_build workunit + extra_system_packages: + deb: + - bison + - flex + - libelf-dev + - libssl-dev + - network-manager + - iproute2 + - util-linux + # for xfstests-dev + - dump + - indent + # for fsx + - libaio-dev + - libtool-bin + - uuid-dev + - xfslibs-dev + # for postgres + - postgresql + - postgresql-client + - postgresql-common + - postgresql-contrib + rpm: + - bison + - flex + - elfutils-libelf-devel + - openssl-devel + - NetworkManager + - iproute + - util-linux + # for xfstests-dev + - libacl-devel + - libaio-devel + - libattr-devel + - libtool + - libuuid-devel + - xfsdump + - xfsprogs + - xfsprogs-devel + # for fsx + - libaio-devel + - libtool + - libuuid-devel + - xfsprogs-devel + # for postgres + - postgresql + - postgresql-server + - postgresql-contrib +syslog: + ignorelist: + - WARNING*.*check_session_state + - WARNING*.*__ceph_remove_cap diff --git a/qa/cephfs/begin/1-ceph.yaml b/qa/cephfs/begin/1-ceph.yaml new file mode 100644 index 000000000..531c8e3e0 --- /dev/null +++ b/qa/cephfs/begin/1-ceph.yaml @@ -0,0 +1,6 @@ +log-rotate: + ceph-mds: 10G + ceph-osd: 10G +tasks: + - ceph: + create_rbd_pool: false diff --git a/qa/cephfs/begin/2-logrotate.yaml b/qa/cephfs/begin/2-logrotate.yaml new file mode 100644 index 000000000..3b876f13b --- /dev/null +++ b/qa/cephfs/begin/2-logrotate.yaml @@ -0,0 +1,3 @@ +log-rotate: + ceph-mds: 10G + ceph-osd: 10G diff --git a/qa/cephfs/clusters/.qa b/qa/cephfs/clusters/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/clusters/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/clusters/1-mds-1-client-coloc.yaml b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml new file mode 100644 index 000000000..d295dc3d3 --- /dev/null +++ b/qa/cephfs/clusters/1-mds-1-client-coloc.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-1-client-micro.yaml b/qa/cephfs/clusters/1-mds-1-client-micro.yaml new file mode 100644 index 000000000..8b66c3906 --- /dev/null +++ b/qa/cephfs/clusters/1-mds-1-client-micro.yaml @@ -0,0 +1,7 @@ +roles: +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3] +- [client.0] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/qa/cephfs/clusters/1-mds-1-client.yaml b/qa/cephfs/clusters/1-mds-1-client.yaml new file mode 100644 index 000000000..d7701815f --- /dev/null +++ b/qa/cephfs/clusters/1-mds-1-client.yaml @@ -0,0 +1,10 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +- [client.0] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-2-client-coloc.yaml b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml new file mode 100644 index 000000000..5ce128cbd --- /dev/null +++ b/qa/cephfs/clusters/1-mds-2-client-coloc.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-2-client-micro.yaml b/qa/cephfs/clusters/1-mds-2-client-micro.yaml new file mode 100644 index 000000000..42ad4efcd --- /dev/null +++ b/qa/cephfs/clusters/1-mds-2-client-micro.yaml @@ -0,0 +1,8 @@ +roles: +- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.a, mds.b, mds.c, osd.0, osd.1, osd.2, osd.3] +- [client.0] +- [client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 10 # GB diff --git a/qa/cephfs/clusters/1-mds-2-client.yaml b/qa/cephfs/clusters/1-mds-2-client.yaml new file mode 100644 index 000000000..6e996244e --- /dev/null +++ b/qa/cephfs/clusters/1-mds-2-client.yaml @@ -0,0 +1,11 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +- [client.0] +- [client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-3-client.yaml b/qa/cephfs/clusters/1-mds-3-client.yaml new file mode 100644 index 000000000..200df1fe0 --- /dev/null +++ b/qa/cephfs/clusters/1-mds-3-client.yaml @@ -0,0 +1,12 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +- [client.0] +- [client.1] +- [client.2] +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-4-client-coloc.yaml b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml new file mode 100644 index 000000000..3e2ee7870 --- /dev/null +++ b/qa/cephfs/clusters/1-mds-4-client-coloc.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0, client.1] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7, client.2, client.3] +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-mds-4-client.yaml b/qa/cephfs/clusters/1-mds-4-client.yaml new file mode 100644 index 000000000..9addfe3b0 --- /dev/null +++ b/qa/cephfs/clusters/1-mds-4-client.yaml @@ -0,0 +1,13 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.b, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mon.c, mgr.x, mds.c, osd.4, osd.5, osd.6, osd.7] +- [client.0] +- [client.1] +- [client.2] +- [client.3] +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml b/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml new file mode 100644 index 000000000..865b976c6 --- /dev/null +++ b/qa/cephfs/clusters/1-node-1-mds-1-osd.yaml @@ -0,0 +1,8 @@ +roles: +- [mon.a, mgr.x, mds.a, osd.0, client.0] +openstack: +- volumes: # attached to each instance + count: 1 + size: 5 # GB +- machine: + disk: 10 # GB diff --git a/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml new file mode 100644 index 000000000..ca6f79ba2 --- /dev/null +++ b/qa/cephfs/clusters/1a11s-mds-1c-client-3node.yaml @@ -0,0 +1,10 @@ +roles: +- [mon.a, mgr.x, mds.a, mds.d, mds.g, mds.j, osd.0, osd.3, osd.6, osd.9, client.0] +- [mon.b, mgr.y, mds.b, mds.e, mds.h, mds.k, osd.1, osd.4, osd.7, osd.10] +- [mon.c, mgr.z, mds.c, mds.f, mds.i, mds.l, osd.2, osd.5, osd.8, osd.11] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml new file mode 100644 index 000000000..3a9ff34f8 --- /dev/null +++ b/qa/cephfs/clusters/1a2s-mds-1c-client-3node.yaml @@ -0,0 +1,10 @@ +roles: +- [mon.a, mgr.x, mds.a, osd.0, osd.3, osd.6, osd.9, client.0] +- [mon.b, mgr.y, mds.b, osd.1, osd.4, osd.7, osd.10] +- [mon.c, mgr.z, mds.c, osd.2, osd.5, osd.8, osd.11] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a3s-mds-1c-client.yaml b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml new file mode 100644 index 000000000..87867daa4 --- /dev/null +++ b/qa/cephfs/clusters/1a3s-mds-1c-client.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a3s-mds-2c-client.yaml b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml new file mode 100644 index 000000000..ba17f05d2 --- /dev/null +++ b/qa/cephfs/clusters/1a3s-mds-2c-client.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7, client.1] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a3s-mds-4c-client.yaml b/qa/cephfs/clusters/1a3s-mds-4c-client.yaml new file mode 100644 index 000000000..51fd809b4 --- /dev/null +++ b/qa/cephfs/clusters/1a3s-mds-4c-client.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, osd.0, osd.1, osd.2, osd.3, client.0, client.2] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, osd.4, osd.5, osd.6, osd.7, client.1, client.3] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml b/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml new file mode 100644 index 000000000..ae723c160 --- /dev/null +++ b/qa/cephfs/clusters/1a5s-mds-1c-client-3node.yaml @@ -0,0 +1,10 @@ +roles: +- [mon.a, mgr.x, mds.a, mds.d, osd.0, osd.3, osd.6, osd.9, client.0] +- [mon.b, mgr.y, mds.b, mds.e, osd.1, osd.4, osd.7, osd.10] +- [mon.c, mgr.z, mds.c, mds.f, osd.2, osd.5, osd.8, osd.11] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/1a5s-mds-1c-client.yaml b/qa/cephfs/clusters/1a5s-mds-1c-client.yaml new file mode 100644 index 000000000..79fd84cf4 --- /dev/null +++ b/qa/cephfs/clusters/1a5s-mds-1c-client.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, mds.c, mds.e, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, mds.d, mds.f, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 20 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/3-mds.yaml b/qa/cephfs/clusters/3-mds.yaml new file mode 100644 index 000000000..8dfacb4e7 --- /dev/null +++ b/qa/cephfs/clusters/3-mds.yaml @@ -0,0 +1,14 @@ +roles: +- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7] +- [client.0, client.1] +overrides: + ceph: + cephfs: + max_mds: 3 +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/9-mds.yaml b/qa/cephfs/clusters/9-mds.yaml new file mode 100644 index 000000000..60b18c251 --- /dev/null +++ b/qa/cephfs/clusters/9-mds.yaml @@ -0,0 +1,14 @@ +roles: +- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3] +- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7] +- [client.0, client.1] +overrides: + ceph: + cephfs: + max_mds: 9 +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/clusters/fixed-2-ucephfs.yaml b/qa/cephfs/clusters/fixed-2-ucephfs.yaml new file mode 100644 index 000000000..dff37a51f --- /dev/null +++ b/qa/cephfs/clusters/fixed-2-ucephfs.yaml @@ -0,0 +1,9 @@ +roles: +- [mon.a, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3, client.0] +- [mon.b, mon.c, mgr.x, mds.b, osd.4, osd.5, osd.6, osd.7] +openstack: +- volumes: # attached to each instance + count: 4 + size: 30 # GB +- machine: + disk: 200 # GB diff --git a/qa/cephfs/conf/+ b/qa/cephfs/conf/+ new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/conf/+ diff --git a/qa/cephfs/conf/.qa b/qa/cephfs/conf/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/conf/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/conf/client.yaml b/qa/cephfs/conf/client.yaml new file mode 100644 index 000000000..4b37d03b4 --- /dev/null +++ b/qa/cephfs/conf/client.yaml @@ -0,0 +1,9 @@ +overrides: + ceph: + conf: + client: + client mount timeout: 600 + debug ms: 1 + debug client: 20 + rados mon op timeout: 900 + rados osd op timeout: 900 diff --git a/qa/cephfs/conf/mds.yaml b/qa/cephfs/conf/mds.yaml new file mode 100644 index 000000000..b1c7a5869 --- /dev/null +++ b/qa/cephfs/conf/mds.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + conf: + mds: + debug mds: 20 + debug mds balancer: 20 + debug ms: 1 + mds debug frag: true + mds debug scatterstat: true + mds op complaint time: 180 + mds verify scatter: true + osd op complaint time: 180 + rados mon op timeout: 900 + rados osd op timeout: 900 diff --git a/qa/cephfs/conf/mon.yaml b/qa/cephfs/conf/mon.yaml new file mode 100644 index 000000000..eea56004a --- /dev/null +++ b/qa/cephfs/conf/mon.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + mon: + mon op complaint time: 120 diff --git a/qa/cephfs/conf/osd.yaml b/qa/cephfs/conf/osd.yaml new file mode 100644 index 000000000..1087202f9 --- /dev/null +++ b/qa/cephfs/conf/osd.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + osd op complaint time: 180 diff --git a/qa/cephfs/mount/.qa b/qa/cephfs/mount/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/fuse.yaml b/qa/cephfs/mount/fuse.yaml new file mode 100644 index 000000000..9ffb5129d --- /dev/null +++ b/qa/cephfs/mount/fuse.yaml @@ -0,0 +1,16 @@ +teuthology: + postmerge: + - local function is_kupstream() + return false + end + - local function is_kdistro() + return false + end + - local function is_fuse() + return true + end + - local function syntax_version() + return '' + end +tasks: + - ceph-fuse: diff --git a/qa/cephfs/mount/kclient/% b/qa/cephfs/mount/kclient/% new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/mount/kclient/% diff --git a/qa/cephfs/mount/kclient/.qa b/qa/cephfs/mount/kclient/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/kclient/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/mount-syntax/$ b/qa/cephfs/mount/kclient/mount-syntax/$ new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/mount/kclient/mount-syntax/$ diff --git a/qa/cephfs/mount/kclient/mount-syntax/v1.yaml b/qa/cephfs/mount/kclient/mount-syntax/v1.yaml new file mode 100644 index 000000000..84d5d43b2 --- /dev/null +++ b/qa/cephfs/mount/kclient/mount-syntax/v1.yaml @@ -0,0 +1,3 @@ +overrides: + kclient: + syntax: 'v1' diff --git a/qa/cephfs/mount/kclient/mount-syntax/v2.yaml b/qa/cephfs/mount/kclient/mount-syntax/v2.yaml new file mode 100644 index 000000000..ef7d30424 --- /dev/null +++ b/qa/cephfs/mount/kclient/mount-syntax/v2.yaml @@ -0,0 +1,3 @@ +overrides: + kclient: + syntax: 'v2' diff --git a/qa/cephfs/mount/kclient/mount.yaml b/qa/cephfs/mount/kclient/mount.yaml new file mode 100644 index 000000000..a130ff9d5 --- /dev/null +++ b/qa/cephfs/mount/kclient/mount.yaml @@ -0,0 +1,16 @@ +teuthology: + postmerge: + - local function is_kupstream() + return yaml.ktype == 'upstream' + end + - local function is_kdistro() + return yaml.ktype == 'distro' + end + - local function is_fuse() + return false + end + - local function syntax_version() + return yaml.overrides.kclient.syntax + end +tasks: +- kclient: diff --git a/qa/cephfs/mount/kclient/overrides/% b/qa/cephfs/mount/kclient/overrides/% new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/% diff --git a/qa/cephfs/mount/kclient/overrides/.qa b/qa/cephfs/mount/kclient/overrides/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/overrides/distro/.qa b/qa/cephfs/mount/kclient/overrides/distro/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/% b/qa/cephfs/mount/kclient/overrides/distro/stock/% new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/stock/% diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/.qa b/qa/cephfs/mount/kclient/overrides/distro/stock/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/stock/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml new file mode 100644 index 000000000..ca2d688bb --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/stock/k-stock.yaml @@ -0,0 +1,4 @@ +kernel: + client: + sha1: distro +ktype: distro diff --git a/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml b/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml new file mode 120000 index 000000000..133acf27b --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/stock/rhel_8.yaml @@ -0,0 +1 @@ +.qa/distros/all/rhel_8.yaml
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/overrides/distro/testing/.qa b/qa/cephfs/mount/kclient/overrides/distro/testing/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/testing/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml b/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml new file mode 100644 index 000000000..2ee219125 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/distro/testing/k-testing.yaml @@ -0,0 +1,4 @@ +kernel: + client: + branch: testing +ktype: upstream diff --git a/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml b/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml new file mode 100644 index 000000000..30da870b2 --- /dev/null +++ b/qa/cephfs/mount/kclient/overrides/ms-die-on-skipped.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + global: + ms die on skipped message: false diff --git a/qa/cephfs/objectstore-ec/.qa b/qa/cephfs/objectstore-ec/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/objectstore-ec/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml b/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml new file mode 120000 index 000000000..9fb86b9fe --- /dev/null +++ b/qa/cephfs/objectstore-ec/bluestore-bitmap.yaml @@ -0,0 +1 @@ +../../objectstore/bluestore-bitmap.yaml
\ No newline at end of file diff --git a/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml b/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml new file mode 100644 index 000000000..512eb117c --- /dev/null +++ b/qa/cephfs/objectstore-ec/bluestore-comp-ec-root.yaml @@ -0,0 +1,29 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + cephfs: + ec_profile: + - m=2 + - k=2 + - crush-failure-domain=osd + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + diff --git a/qa/cephfs/objectstore-ec/bluestore-comp.yaml b/qa/cephfs/objectstore-ec/bluestore-comp.yaml new file mode 100644 index 000000000..b408032fd --- /dev/null +++ b/qa/cephfs/objectstore-ec/bluestore-comp.yaml @@ -0,0 +1,23 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore compression mode: aggressive + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true diff --git a/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml b/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml new file mode 100644 index 000000000..b89c4c711 --- /dev/null +++ b/qa/cephfs/objectstore-ec/bluestore-ec-root.yaml @@ -0,0 +1,43 @@ +overrides: + thrashosds: + bdev_inject_crash: 2 + bdev_inject_crash_probability: .5 + ceph: + fs: xfs + cephfs: + ec_profile: + - m=2 + - k=2 + - crush-failure-domain=osd + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 +# this doesn't work with failures bc the log writes are not atomic across the two backends +# bluestore bluefs env mirror: true + ceph-deploy: + fs: xfs + bluestore: yes + conf: + osd: + osd objectstore: bluestore + bluestore block size: 96636764160 + debug bluestore: 20 + debug bluefs: 20 + debug rocksdb: 10 + bluestore fsck on mount: true + # lower the full ratios since we can fill up a 100gb osd so quickly + mon osd full ratio: .9 + mon osd backfillfull_ratio: .85 + mon osd nearfull ratio: .8 + osd failsafe full ratio: .95 + diff --git a/qa/cephfs/overrides/.qa b/qa/cephfs/overrides/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/overrides/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/overrides/frag.yaml b/qa/cephfs/overrides/frag.yaml new file mode 100644 index 000000000..f05b3f48f --- /dev/null +++ b/qa/cephfs/overrides/frag.yaml @@ -0,0 +1,9 @@ +overrides: + ceph: + conf: + mds: + mds bal fragment size max: 10000 + mds bal merge size: 5 + mds bal split bits: 3 + mds bal split size: 100 + diff --git a/qa/cephfs/overrides/fuse/.qa b/qa/cephfs/overrides/fuse/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/overrides/fuse/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/overrides/fuse/default-perm/% b/qa/cephfs/overrides/fuse/default-perm/% new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/qa/cephfs/overrides/fuse/default-perm/% diff --git a/qa/cephfs/overrides/fuse/default-perm/.qa b/qa/cephfs/overrides/fuse/default-perm/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/overrides/fuse/default-perm/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/overrides/fuse/default-perm/no.yaml b/qa/cephfs/overrides/fuse/default-perm/no.yaml new file mode 100644 index 000000000..445e93652 --- /dev/null +++ b/qa/cephfs/overrides/fuse/default-perm/no.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + client: + fuse default permissions: false diff --git a/qa/cephfs/overrides/fuse/default-perm/yes.yaml b/qa/cephfs/overrides/fuse/default-perm/yes.yaml new file mode 100644 index 000000000..2fd210a39 --- /dev/null +++ b/qa/cephfs/overrides/fuse/default-perm/yes.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + client: + fuse default permissions: true diff --git a/qa/cephfs/overrides/ignorelist_health.yaml b/qa/cephfs/overrides/ignorelist_health.yaml new file mode 100644 index 000000000..d8b819288 --- /dev/null +++ b/qa/cephfs/overrides/ignorelist_health.yaml @@ -0,0 +1,13 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_DEGRADED\) + - \(MDS_FAILED\) + - \(MDS_DEGRADED\) + - \(FS_WITH_FAILED_MDS\) + - \(MDS_DAMAGE\) + - \(MDS_ALL_DOWN\) + - \(MDS_UP_LESS_THAN_MAX\) + - \(FS_INLINE_DATA_DEPRECATED\) + - \(POOL_APP_NOT_ENABLED\) diff --git a/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml new file mode 100644 index 000000000..41ba84f04 --- /dev/null +++ b/qa/cephfs/overrides/ignorelist_wrongly_marked_down.yaml @@ -0,0 +1,9 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(OSD_DOWN\) + - \(OSD_ + - but it is still running +# MDS daemon 'b' is not responding, replacing it as rank 0 with standby 'a' + - is not responding diff --git a/qa/cephfs/overrides/osd-asserts.yaml b/qa/cephfs/overrides/osd-asserts.yaml new file mode 100644 index 000000000..8c16e6e1c --- /dev/null +++ b/qa/cephfs/overrides/osd-asserts.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + osd: + osd shutdown pgref assert: true diff --git a/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml b/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml new file mode 100644 index 000000000..71d6d73ba --- /dev/null +++ b/qa/cephfs/overrides/prefetch_entire_dirfrags/no.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + mds: + mds_dir_prefetch: false diff --git a/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml b/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml new file mode 100644 index 000000000..5d25b61d1 --- /dev/null +++ b/qa/cephfs/overrides/prefetch_entire_dirfrags/yes.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + mds: + mds_dir_prefetch: true diff --git a/qa/cephfs/overrides/session_timeout.yaml b/qa/cephfs/overrides/session_timeout.yaml new file mode 100644 index 000000000..a7a163337 --- /dev/null +++ b/qa/cephfs/overrides/session_timeout.yaml @@ -0,0 +1,4 @@ +overrides: + ceph: + cephfs: + session_timeout: 300 diff --git a/qa/cephfs/tasks/.qa b/qa/cephfs/tasks/.qa new file mode 120000 index 000000000..a602a0353 --- /dev/null +++ b/qa/cephfs/tasks/.qa @@ -0,0 +1 @@ +../.qa/
\ No newline at end of file diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml new file mode 100644 index 000000000..2d370d7ef --- /dev/null +++ b/qa/cephfs/tasks/cfuse_workunit_suites_blogbench.yaml @@ -0,0 +1,9 @@ +tasks: +- check-counter: + counters: + mds: + - "mds.dir_split" +- workunit: + clients: + all: + - suites/blogbench.sh diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml new file mode 100644 index 000000000..41b2bc8ed --- /dev/null +++ b/qa/cephfs/tasks/cfuse_workunit_suites_dbench.yaml @@ -0,0 +1,5 @@ +tasks: +- workunit: + clients: + all: + - suites/dbench.sh diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml new file mode 100644 index 000000000..6a2b35a18 --- /dev/null +++ b/qa/cephfs/tasks/cfuse_workunit_suites_ffsb.yaml @@ -0,0 +1,17 @@ +overrides: + ceph: + log-ignorelist: + - SLOW_OPS + - slow request + conf: + osd: + filestore flush min: 0 +tasks: +- check-counter: + counters: + mds: + - "mds.dir_split" +- workunit: + clients: + all: + - suites/ffsb.sh diff --git a/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml b/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml new file mode 100644 index 000000000..bae220292 --- /dev/null +++ b/qa/cephfs/tasks/cfuse_workunit_suites_fsstress.yaml @@ -0,0 +1,6 @@ +tasks: +- workunit: + timeout: 6h + clients: + all: + - suites/fsstress.sh diff --git a/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml b/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml new file mode 100644 index 000000000..e51542022 --- /dev/null +++ b/qa/cephfs/tasks/cfuse_workunit_trivial_sync.yaml @@ -0,0 +1,5 @@ +tasks: +- workunit: + clients: + all: + - fs/misc/trivial_sync.sh diff --git a/qa/cephfs/tasks/libcephfs_interface_tests.yaml b/qa/cephfs/tasks/libcephfs_interface_tests.yaml new file mode 100644 index 000000000..c59775259 --- /dev/null +++ b/qa/cephfs/tasks/libcephfs_interface_tests.yaml @@ -0,0 +1,14 @@ +overrides: + ceph-fuse: + disabled: true + kclient: + disabled: true +tasks: +- check-counter: + counters: + mds: + - "mds.dir_split" +- workunit: + clients: + client.0: + - libcephfs/test.sh diff --git a/qa/cephfs/unshare_ns_mount.sh b/qa/cephfs/unshare_ns_mount.sh new file mode 100755 index 000000000..88ac3e933 --- /dev/null +++ b/qa/cephfs/unshare_ns_mount.sh @@ -0,0 +1,594 @@ +#!/usr/bin/env bash + +# This is one helper for mounting the ceph-fuse/kernel clients by +# unsharing the network namespace, let's call it netns container. +# With the netns container, you can easily suspend or resume the +# virtual network interface to simulate the client node hard +# shutdown for some test cases. +# +# netnsX netnsY netnsZ +# -------------- -------------- -------------- +# | mount client | | mount client | | mount client | +# | default | ... | default | ... | default | +# |192.168.0.1/16| |192.168.0.2/16| |192.168.0.3/16| +# | veth0 | | veth0 | | veth0 | +# -------------- -------------- ------------- +# | | | +# \ | brx.Y / +# \ ---------------------- / +# \ brx.X | ceph-brx | brx.Z / +# \------>| default |<------/ +# | | 192.168.255.254/16 | | +# | ---------------------- | +# (suspend/resume) | (suspend/resume) +# ----------- +# | Physical | +# | A.B.C.D/M | +# ----------- +# +# Defaultly it will use the 192.168.X.Y/16 private network IPs for +# the ceph-brx and netnses as above. And you can also specify your +# own new ip/mask for the ceph-brx, like: +# +# $ unshare_ns_mount.sh --fuse /mnt/cephfs --brxip 172.19.100.100/12 +# +# Then the each netns will get a new ip from the ranges: +# [172.16.0.1 ~ 172.19.100.99]/12 and [172.19.100.101 ~ 172.31.255.254]/12 + +usage() { + echo "" + echo "This will help to isolate the network namespace from OS for the mount client!" + echo "" + echo "usage: unshare_ns_mount.sh [OPTIONS [paramters]] [--brxip <ip_address/mask>]" + echo "OPTIONS:" + echo -e " --fuse <ceph-fuse options>" + echo -e "\tThe ceph-fuse command options" + echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs -o nonempty" + echo "" + echo -e " --kernel <mount options>" + echo -e "\tThe mount command options" + echo -e "\t $ unshare_ns_mount.sh --kernel -t ceph 192.168.0.1:6789:/ /mnt/cephfs -o fs=a" + echo "" + echo -e " --suspend <mountpoint>" + echo -e "\tDown the veth interface in the network namespace" + echo -e "\t $ unshare_ns_mount.sh --suspend /mnt/cephfs" + echo "" + echo -e " --resume <mountpoint>" + echo -e "\tUp the veth interface in the network namespace" + echo -e "\t $ unshare_ns_mount.sh --resume /mnt/cephfs" + echo "" + echo -e " --umount <mountpoint>" + echo -e "\tUmount and delete the network namespace" + echo -e "\t $ unshare_ns_mount.sh --umount /mnt/cephfs" + echo "" + echo -e " --brxip <ip_address/mask>" + echo -e "\tSpecify ip/mask for ceph-brx and it only makes sense for --fuse/--kernel options" + echo -e "\t(default: 192.168.255.254/16, netns ip: 192.168.0.1/16 ~ 192.168.255.253/16)" + echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs --brxip 172.19.255.254/12" + echo -e "\t $ unshare_ns_mount.sh --kernel 192.168.0.1:6789:/ /mnt/cephfs --brxip 172.19.255.254/12" + echo "" + echo -e " -h, --help" + echo -e "\tPrint help" + echo "" +} + +CEPH_BRX=ceph-brx +CEPH_BRX_IP_DEF=192.168.255.254 +NET_MASK_DEF=16 +BRD_DEF=192.168.255.255 + +CEPH_BRX_IP=$CEPH_BRX_IP_DEF +NET_MASK=$NET_MASK_DEF +BRD=$BRD_DEF + +mountpoint="" +new_netns="" +fuse_type=false + +function get_mountpoint() { + for param in $@ + do + if [ -d $param ]; then + # skipping "--client_mountpoint/-r root_directory" + # option for ceph-fuse command + if [ "$last" == "-r" -o "$last" == "--client_mountpoint" ]; then + last=$param + continue + fi + if [ "0$mountpoint" != "0" ]; then + echo "Oops: too many mountpiont options!" + exit 1 + fi + mountpoint=$param + fi + last=$param + done + + if [ "0$mountpoint" == "0" ]; then + echo "Oops: mountpoint path is not a directory or no mountpoint specified!" + exit 1 + fi +} + +function get_new_netns() { + # prune the repeating slashes: + # "/mnt///cephfs///" --> "/mnt/cephfs/" + __mountpoint=`echo "$mountpoint" | sed 's/\/\+/\//g'` + + # prune the leading slashes + while [ ${__mountpoint:0:1} == "/" ] + do + __mountpoint=${__mountpoint:1} + done + + # prune the last slashes + while [ ${__mountpoint: -1} == "/" ] + do + __mountpoint=${__mountpoint:0:-1} + done + + # replace '/' with '-' + __mountpoint=${__mountpoint//\//-} + + # "mnt/cephfs" --> "ceph-fuse-mnt-cephfs" + if [ "$1" == "--fuse" ]; then + new_netns=`echo ceph-fuse-$__mountpoint` + fuse_type=true + return + fi + + # "mnt/cephfs" --> "ceph-kernel-mnt-cephfs" + if [ "$1" == "--kernel" ]; then + new_netns=`echo ceph-kernel-$__mountpoint` + return + fi + + # we are in umount/suspend/resume routines + for ns in `ip netns list | awk '{print $1}'` + do + if [ "$ns" == "ceph-fuse-$__mountpoint" ]; then + new_netns=$ns + fuse_type=true + return + fi + if [ "$ns" == "ceph-kernel-$__mountpoint" ]; then + new_netns=$ns + return + fi + done + + if [ "0$new_netns" == "0" ]; then + echo "Oops, netns 'ceph-{fuse/kernel}-$__mountpoint' does not exists!" + exit 1 + fi +} + +# the peer veth name will be "brx.$nsid" on host node +function get_netns_brx() { + get_new_netns + + nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'` + netns_veth=brx.$nsid + eval $1="$netns_veth" +} + +function suspend_netns_veth() { + get_mountpoint $@ + + get_netns_brx brx + ip link set $brx down + exit 0 +} + +function resume_netns_veth() { + get_mountpoint $@ + + get_netns_brx brx + ip link set $brx up + exit 0 +} + +# help and usage +if [ $# == 0 -o "$1" == "-h" -o "$1" == "--help" ]; then + usage + exit 0 +fi + +# suspend the veth from network namespace +if [ $1 == "--suspend" ]; then + suspend_netns_veth $@ + exit 0 +fi + +# resume the veth from network namespace +if [ $1 == "--resume" ]; then + resume_netns_veth $@ + exit 0 +fi + +function ceph_umount() { + get_mountpoint $@ + get_new_netns + + if [ $fuse_type == true ]; then + nsenter --net=/var/run/netns/$new_netns fusermount -u $mountpoint 2>/dev/null + else + nsenter --net=/var/run/netns/$new_netns umount $mountpoint 2>/dev/null + fi + + # let's wait for a while to let the umount operation + # to finish before deleting the netns + while [ 1 ] + do + for pid in `ip netns pids $new_netns 2>/dev/null` + do + name=`cat /proc/$pid/comm 2>/dev/null` + if [ "$name" == "ceph-fuse" ]; then + break + fi + done + + if [ "$name" == "ceph-fuse" ]; then + name="" + usleep 100000 + continue + fi + + break + done + + nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'` + netns_brx=brx.$nsid + + # brctl delif $CEPH_BRX $netns_brx 2>/dev/null + nmcli connection down $netns_brx down 2>/dev/null + nmcli connection delete $netns_brx 2>/dev/null + + ip netns delete $new_netns 2>/dev/null + + # if this is the last netns_brx, will delete + # the $CEPH_BRX and restore the OS configure + # rc=`brctl show ceph-brx 2>/dev/null | grep 'brx\.'|wc -l` + rc=`nmcli connection show 2>/dev/null | grep 'brx\.' | wc -l` + if [ $rc == 0 ]; then + ip link set $CEPH_BRX down 2>/dev/null + # brctl delbr $CEPH_BRX 2>/dev/null + nmcli connection delete $CEPH_BRX 2>/dev/null + + # restore the ip forward + tmpfile=`ls /tmp/ | grep "$CEPH_BRX\."` + tmpfile=/tmp/$tmpfile + if [ ! -f $tmpfile ]; then + echo "Oops, the $CEPH_BRX.XXX temp file does not exist!" + else + save=`cat $tmpfile` + echo $save > /proc/sys/net/ipv4/ip_forward + rm -rf $tmpfile + fi + + # drop the iptables NAT rules + host_nic=`route | grep default | awk '{print $8}'` + iptables -D FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT + iptables -D FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT + iptables -t nat -D POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE + fi +} + +function get_brd_mask() { + first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'` + second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'` + third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'` + fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'` + + if [ "$first" == "172" ]; then + second_max=31 + else + second_max=255 + fi + third_max=255 + fourth_max=255 + + if [ $NET_MASK -lt 16 ]; then + let power=16-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + second=$((second&~m)) + let second_max=$second+$m + elif [ $NET_MASK -lt 24 ]; then + let power=24-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + third=$((third&~m)) + let third_max=$third+$m + second_max=$second + elif [ $NET_MASK -lt 32 ]; then + let power=32-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + fourth=$((fourth&~m)) + let fourth_max=$fourth+$m + second_max=$second + third_max=$third + fi + + BRD=$first.$second_max.$third_max.$fourth_max +} + +# As default: +# The netns IP will be 192.168.0.1 ~ 192.168.255.253, +# and 192.168.255.254 is saved for $CEPH_BRX +function get_new_ns_ip() { + first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'` + second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'` + third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'` + fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'` + + if [ "$first" == ""172 ]; then + second_max=31 + else + second_max=255 + fi + third_max=255 + fourth_max=254 + + if [ $NET_MASK -lt 16 ]; then + let power=16-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + second=$((second&~m)) + let second_max=$second+$m + third=0 + fourth=1 + elif [ $NET_MASK -lt 24 ]; then + let power=24-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + third=$((third&~m)) + let third_max=$third+$m + second_max=$second + fourth=1 + elif [ $NET_MASK -lt 32 ]; then + let power=32-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + fourth=$((fourth&~m)) + let fourth+=1 + let fourth_max=$fourth+$m-1 + second_max=$second + third_max=$third + fi + + while [ $second -le $second_max -a $third -le $third_max -a $fourth -le $fourth_max ] + do + conflict=false + + # check from the existing network namespaces + for netns in `ip netns list | awk '{print $1}'` + do + ip=`ip netns exec $netns ip addr | grep "inet " | grep "veth0"` + ip=`echo "$ip" | awk '{print $2}' | awk -F/ '{print $1}'` + if [ "0$ip" == "0" ]; then + continue + fi + if [ "$first.$second.$third.$fourth" == "$ip" ]; then + conflict=true + + let fourth+=1 + if [ $fourth -le $fourth_max ]; then + break + fi + + fourth=0 + let third+=1 + if [ $third -le $third_max ]; then + break + fi + + third=0 + let second+=1 + if [ $second -le $second_max ]; then + break + fi + + echo "Oops: we have ran out of the ip addresses!" + exit 1 + fi + done + + # have we found one ? + if [ $conflict == false ]; then + break + fi + done + + ip=$first.$second.$third.$fourth + max=$first.$second_max.$third_max.$fourth_max + if [ "$ip" == "$max" ]; then + echo "Oops: we have ran out of the ip addresses!" + exit 1 + fi + + eval $1="$ip" +} + +function check_valid_private_ip() { + first=`echo "$1" | awk -F. '{print $1}'` + second=`echo "$1" | awk -F. '{print $2}'` + + # private network class A 10.0.0.0 - 10.255.255.255 + if [ "$first" == "10" -a $NET_MASK -ge 8 ]; then + return + fi + + # private network class B 172.16.0.0 - 172.31.255.255 + if [ "$first" == "172" -a $second -ge 16 -a $second -le 31 -a $NET_MASK -ge 12 ]; then + return + fi + + # private network class C 192.168.0.0 - 192.168.255.255 + if [ "$first" == "192" -a "$second" == "168" -a $NET_MASK -ge 16 ]; then + return + fi + + echo "Oops: invalid private ip address '$CEPH_BRX_IP/$NET_MASK'!" + exit 1 +} + +function setup_bridge_and_nat() { + # check and parse the --brxip parameter + is_brxip=false + for ip in $@ + do + if [ "$ip" == "--brxip" ]; then + is_brxip=true + continue + fi + if [ $is_brxip == true ]; then + new_brxip=$ip + break + fi + done + + # if the $CEPH_BRX already exists, then check the new + # brxip, if not match fail it without doing anything. + rc=`ip addr | grep "inet " | grep " $CEPH_BRX"` + if [ "0$rc" != "0" ]; then + existing_brxip=`echo "$rc" | awk '{print $2}'` + if [ "0$new_brxip" != "0" -a "$existing_brxip" != "$new_brxip" ]; then + echo "Oops: conflict with the existing $CEPH_BRX ip '$existing_brxip', new '$new_brxip'!" + exit 1 + fi + + CEPH_BRX_IP=`echo "$existing_brxip" | awk -F/ '{print $1}'` + NET_MASK=`echo "$existing_brxip" | awk -F/ '{print $2}'` + get_brd_mask + return + fi + + # if it is the first time to run the the script or there + # is no any network namespace exists, we need to setup + # the $CEPH_BRX, if no --brxip is specified will use the + # default $CEPH_BRX_IP/$NET_MASK + if [ "0$new_brxip" != "0" ]; then + CEPH_BRX_IP=`echo "$new_brxip" | awk -F/ '{print $1}'` + NET_MASK=`echo "$new_brxip" | awk -F/ '{print $2}'` + get_brd_mask + check_valid_private_ip $CEPH_BRX_IP + fi + + # brctl addbr $CEPH_BRX + nmcli connection add type bridge con-name $CEPH_BRX ifname $CEPH_BRX stp no + # ip link set $CEPH_BRX up + # ip addr add $CEPH_BRX_IP/$NET_MASK brd $BRD dev $CEPH_BRX + nmcli connection modify $CEPH_BRX ipv4.addresses $CEPH_BRX_IP/$NET_MASK ipv4.method manual + nmcli connection up $CEPH_BRX + + # setup the NAT + rm -rf /tmp/ceph-brx.* + tmpfile=$(mktemp /tmp/ceph-brx.XXXXXXXX) + save=`cat /proc/sys/net/ipv4/ip_forward` + echo $save > $tmpfile + echo 1 > /proc/sys/net/ipv4/ip_forward + + host_nic=`route | grep default | awk '{print $8}'` + iptables -A FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT + iptables -A FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT + iptables -t nat -A POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE +} + +function __ceph_mount() { + # for some options like the '-t' in mount command + # the nsenter command will take over it, so it is + # hard to pass it direct to the netns. + # here we will create one temp file with x mode + tmpfile=$(mktemp /tmp/ceph-nsenter.XXXXXXXX) + chmod +x $tmpfile + if [ "$1" == "--kernel" ]; then + cmd=`echo "$@" | sed 's/--kernel/mount/'` + else + cmd=`echo "$@" | sed 's/--fuse/ceph-fuse/'` + fi + + # remove the --brxip parameter + cmd=`echo "$cmd" | sed 's/--brxip.*\/[0-9]* //'` + + # enter $new_netns and run ceph fuse client mount, + # we couldn't use 'ip netns exec' here because it + # will unshare the mount namespace. + echo "$cmd" > $tmpfile + nsenter --net=/var/run/netns/$new_netns /bin/bash $tmpfile ; echo $? > $tmpfile + rc=`cat $tmpfile` + rm -f $tmpfile + + # fall back + if [ $rc != 0 ]; then + m=$mountpoint + mountpoint="" + ceph_umount $m + fi +} + +function get_new_nsid() { + # get one uniq netns id + uniq_id=0 + while [ 1 ] + do + rc=`ip netns list-id | grep "nsid $uniq_id "` + if [ "0$rc" == "0" ]; then + break + fi + let uniq_id+=1 + done + + eval $1="$uniq_id" +} + +function ceph_mount() { + get_mountpoint $@ + setup_bridge_and_nat $@ + + get_new_netns $1 + rc=`ip netns list | grep "$new_netns" | awk '{print $1}'` + if [ "0$rc" != "0" ]; then + echo "Oops: the netns "$new_netns" already exists!" + exit 1 + fi + + get_new_nsid new_nsid + + # create a new network namespace + ip netns add $new_netns + ip netns set $new_netns $new_nsid + + get_new_ns_ip ns_ip + if [ 0"$ns_ip" == "0" ]; then + echo "Oops: there is no ip address could be used any more!" + exit 1 + fi + + # veth interface in netns + ns_veth=veth0 + netns_brx=brx.$new_nsid + + # setup veth interfaces + ip link add $ns_veth netns $new_netns type veth peer name $netns_brx + ip netns exec $new_netns ip addr add $ns_ip/$NET_MASK brd $BRD dev $ns_veth + ip netns exec $new_netns ip link set $ns_veth up + ip netns exec $new_netns ip link set lo up + ip netns exec $new_netns ip route add default via $CEPH_BRX_IP + + # bring up the bridge interface and join it to $CEPH_BRX + # brctl addif $CEPH_BRX $netns_brx + nmcli connection add type bridge-slave con-name $netns_brx ifname $netns_brx master $CEPH_BRX + nmcli connection up $netns_brx + # ip link set $netns_brx up + + __ceph_mount $@ +} + +if [ "$1" == "--umount" ]; then + ceph_umount $@ + exit 0 +fi + +# mount in the netns +if [ "$1" != "--kernel" -a "$1" != "--fuse" ]; then + echo "Oops: invalid mount options '$1'!" + exit 1 +fi + +ceph_mount $@ |