blob: af79e3f7d3ba3d2771b736e33979c3ffd580fb45 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
#!/usr/bin/env bash
NVMF_PORT=4420
NVMF_IP_PREFIX="192.168.100"
NVMF_IP_LEAST_ADDR=8
if [ -z "$NVMF_APP" ]; then
NVMF_APP=./app/nvmf_tgt/nvmf_tgt
fi
if [ -z "$NVMF_TEST_CORE_MASK" ]; then
NVMF_TEST_CORE_MASK=0xFF
fi
function load_ib_rdma_modules()
{
if [ `uname` != Linux ]; then
return 0
fi
modprobe ib_cm
modprobe ib_core
# Newer kernels do not have the ib_ucm module
modprobe ib_ucm || true
modprobe ib_umad
modprobe ib_uverbs
modprobe iw_cm
modprobe rdma_cm
modprobe rdma_ucm
}
function detect_soft_roce_nics()
{
if hash rxe_cfg; then
rxe_cfg start
rdma_nics=$(get_rdma_if_list)
all_nics=$(ip -o link | awk '{print $2}' | cut -d":" -f1)
non_rdma_nics=$(echo -e "$rdma_nics\n$all_nics" | sort | uniq -u)
for nic in $non_rdma_nics; do
if [[ -d /sys/class/net/${nic}/bridge ]]; then
continue
fi
rxe_cfg add $nic || true
done
fi
}
function detect_mellanox_nics()
{
if ! hash lspci; then
echo "No NICs"
return 0
fi
nvmf_nic_bdfs=`lspci | grep Ethernet | grep Mellanox | awk -F ' ' '{print "0000:"$1}'`
mlx_core_driver="mlx4_core"
mlx_ib_driver="mlx4_ib"
mlx_en_driver="mlx4_en"
if [ -z "$nvmf_nic_bdfs" ]; then
echo "No NICs"
return 0
fi
# for nvmf target loopback test, suppose we only have one type of card.
for nvmf_nic_bdf in $nvmf_nic_bdfs
do
result=`lspci -vvv -s $nvmf_nic_bdf | grep 'Kernel modules' | awk -F ' ' '{print $3}'`
if [ "$result" == "mlx5_core" ]; then
mlx_core_driver="mlx5_core"
mlx_ib_driver="mlx5_ib"
mlx_en_driver=""
fi
break;
done
modprobe $mlx_core_driver
modprobe $mlx_ib_driver
if [ -n "$mlx_en_driver" ]; then
modprobe $mlx_en_driver
fi
# The mlx4 driver takes an extra few seconds to load after modprobe returns,
# otherwise iproute2 operations will do nothing.
sleep 5
}
function detect_rdma_nics()
{
nics=$(detect_mellanox_nics)
if [ "$nics" == "No NICs" ]; then
detect_soft_roce_nics
fi
}
function allocate_nic_ips()
{
let count=$NVMF_IP_LEAST_ADDR
for nic_name in $(get_rdma_if_list); do
ip="$(get_ip_address $nic_name)"
if [ -z $ip ]; then
ip addr add $NVMF_IP_PREFIX.$count/24 dev $nic_name
ip link set $nic_name up
let count=$count+1
fi
# dump configuration for debug log
ip addr show $nic_name
done
}
function get_available_rdma_ips()
{
for nic_name in $(get_rdma_if_list); do
get_ip_address $nic_name
done
}
function get_rdma_if_list()
{
for nic_type in `ls /sys/class/infiniband`; do
for nic_name in `ls /sys/class/infiniband/${nic_type}/device/net`; do
echo "$nic_name"
done
done
}
function get_ip_address()
{
interface=$1
ip -o -4 addr show $interface | awk '{print $4}' | cut -d"/" -f1
}
function nvmfcleanup()
{
sync
set +e
for i in {1..20}; do
modprobe -v -r nvme-rdma nvme-fabrics
if [ $? -eq 0 ]; then
set -e
return
fi
sleep 1
done
set -e
# So far unable to remove the kernel modules. Try
# one more time and let it fail.
modprobe -v -r nvme-rdma nvme-fabrics
}
function nvmftestinit()
{
if [ "$1" == "iso" ]; then
$rootdir/scripts/setup.sh
rdma_device_init
fi
}
function nvmftestfini()
{
if [ "$1" == "iso" ]; then
$rootdir/scripts/setup.sh reset
rdma_device_init
fi
}
function rdma_device_init()
{
load_ib_rdma_modules
detect_rdma_nics
allocate_nic_ips
}
function revert_soft_roce()
{
if hash rxe_cfg; then
interfaces="$(ip -o link | awk '{print $2}' | cut -d":" -f1)"
for interface in $interfaces; do
rxe_cfg remove $interface || true
done
rxe_cfg stop || true
fi
}
function check_ip_is_soft_roce()
{
IP=$1
if hash rxe_cfg; then
dev=$(ip -4 -o addr show | grep $IP | cut -d" " -f2)
if rxe_cfg | grep $dev; then
return 0
else
return 1
fi
else
return 1
fi
}
|
