blob: 57447b957d789a357b8d79eb06172e09c96a2c0f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
#!/bin/bash
set -ex
FIRST_DAMAGE="first-damage.py"
FS=cephfs
METADATA_POOL=cephfs_meta
MOUNT=~/mnt/mnt.0
PYTHON=python3
function usage {
printf '%s: [--fs=<fs_name>] [--metadata-pool=<pool>] [--first-damage=</path/to/first-damage.py>]\n'
exit 1
}
function create {
ceph config set mds mds_bal_fragment_dirs 0
mkdir dir
DIR_INODE=$(stat -c '%i' dir)
touch dir/a
touch dir/"a space"
touch -- $(printf 'dir/\xff')
mkdir dir/.snap/1
mkdir dir/.snap/2
# two snaps
rm dir/a
mkdir dir/.snap/3
# not present in HEAD
touch dir/a
mkdir dir/.snap/4
# one snap
rm dir/a
touch dir/a
mkdir dir/.snap/5
# unlink then create
rm dir/a
touch dir/a
# unlink then create, HEAD not snapped
ls dir/.snap/*/
mkdir big
BIG_DIR_INODE=$(stat -c '%i' big)
for i in `seq 1 15000`; do
touch $(printf 'big/%08d' $i)
done
}
function flush {
ceph tell mds."$FS":0 flush journal
}
function damage {
local IS=$(printf '%llx.%08llx' "$DIR_INODE" 0)
local LS=$(ceph tell mds."$FS":0 dump snaps | jq .last_created)
local T=$(mktemp -p /tmp)
# nuke snap 1 version of "a"
rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-4))) "$T"
printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-4))) --input-file="$T"
# nuke snap 4 version of "a"
rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-1))) "$T"
printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-1))) --input-file="$T"
# screw up HEAD
rados --pool="$METADATA_POOL" getomapval "$IS" a_head "$T"
printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
rados --pool="$METADATA_POOL" setomapval "$IS" a_head --input-file="$T"
# screw up HEAD on what dentry in big
IS=$(printf '%llx.%08llx' "$BIG_DIR_INODE" 0)
rados --pool="$METADATA_POOL" getomapval "$IS" 00009999_head "$T"
printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat
rados --pool="$METADATA_POOL" setomapval "$IS" 00009999_head --input-file="$T"
rm -f "$T"
}
function recover {
flush
ceph fs fail "$FS"
sleep 5
cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
cephfs-journal-tool --rank="$FS":0 journal reset
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL"
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
ceph fs set "$FS" joinable true
}
function check {
stat dir || exit 1
stat dir/a || exit 1
for i in `seq 1 5`; do
stat dir/.snap/$i || exit 2
done
stat dir/.snap/2/a || exit 3
stat dir/.snap/5/a || exit 4
if stat dir/.snap/1/a; then
echo should be gone
exit 5
fi
if stat dir/.snap/3/a; then
echo should not ever exist
exit 6
fi
if stat dir/.snap/4/a; then
echo should be gone
exit 7
fi
}
function cleanup {
rmdir dir/.snap/*
find dir
rm -rf dir
}
function mount {
sudo --preserve-env=CEPH_CONF bin/mount.ceph :/ "$MOUNT" -o name=admin,noshare
df -h "$MOUNT"
}
function main {
eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:,python:' -- "$@")
while [ "$#" -gt 0 ]; do
echo "$*"
echo "$1"
case "$1" in
-h|--help)
usage
;;
--fs)
FS="$2"
shift 2
;;
--metadata-pool)
METADATA_POOL="$2"
shift 2
;;
--mount)
MOUNT="$2"
shift 2
;;
--first-damage)
FIRST_DAMAGE="$2"
shift 2
;;
--python)
PYTHON="$2"
shift 2
;;
--)
shift
break
;;
*)
usage
;;
esac
done
mount
pushd "$MOUNT"
create
popd
sudo umount -f "$MOUNT"
# flush dentries/inodes to omap
flush
damage
recover
sleep 5 # for mds to join
mount
pushd "$MOUNT"
check
cleanup
popd
sudo umount -f "$MOUNT"
}
main "$@"
|
