forked from awslabs/libfabric-ci-scripts
-
Notifications
You must be signed in to change notification settings - Fork 0
/
efa-check.sh
executable file
·245 lines (219 loc) · 7.02 KB
/
efa-check.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
#!/bin/bash
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Script to verify EFA configuration and assist with debugging.
source ~/wget_check.sh
VENDOR_ID="0x1d0f"
ami_arch="$(uname -m)"
if [ "${ami_arch}" == "x86_64" ]; then
DEV_ID="0xefa0"
elif [ "${ami_arch}" == "aarch64" ]; then
DEV_ID="0xefa1"
else
echo "Unknown architecture, exiting."
fi
usage() {
cat << EOF
usage: $(basename "$0") [options]
Options:
--skip-libfabric Skip libfabric checks
--skip-mpi Skip mpi checks
EOF
}
libfabric_checks() {
# Check for libfabric and print the version.
libfabric=$(sudo ldconfig -p | tr -d '\t' | grep '^libfabric.so.1')
if [ $? -ne 0 ]; then
cat >&2 << EOF
Error: libfabric shared library not found.
EOF
return 1
fi
echo "libfabric in ldcache: $(readlink -m "$(echo "$libfabric" | awk '{print $4}')")"
if command -v fi_info >/dev/null 2>&1; then
echo "libfabric version:"
fi_info --version
echo "EFA libfabric providers:"
fi_info -p efa
if [ "$efa_gdr_enabled" -eq 1 ]; then
if ! FI_EFA_USE_DEVICE_RDMA=1 fi_info -p efa -c FI_HMEM; then
echo "EFA libfabric provider does not have FI_HMEM capability."
return 1
else
echo "EFA libfabric provider has FI_HMEM capability."
fi
fi
fi
}
mpi_checks() {
# Print location of mpirun and its version
mpirun=$(command -v mpirun)
if [ $? -ne 0 ]; then
cat >&2 << EOF
Warning: mpirun not found in \$PATH.
EOF
return 1
else
echo "Current mpirun in \$PATH: $mpirun"
$mpirun --version | grep -v "^Report"
fi
}
ret=0
skip_libfabric=0
skip_mpi=0
for arg in "$@"; do
case "$arg" in
--skip-libfabric)
skip_libfabric=1
;;
--skip-mpi)
skip_mpi=1
;;
-h|--help)
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
echo "======== Instance / Device check ========"
# Get instance type
if command -v curl >/dev/null 2>&1; then
metadata_url="http://169.254.169.254/latest/meta-data/instance-type"
wget_check "$metadata_url" "instance-type"
instance=$(cat instance-type)
echo "Instance type: ${instance}"
fi
# Determine if an EFA device is present and print device list.
efa_detected=0
for dev in /sys/class/infiniband/*/device; do
if [ "$(cat "${dev}"/subsystem_vendor)" = "$VENDOR_ID" ] && \
[ "$(cat "${dev}"/subsystem_device)" = "$DEV_ID" ]; then
efa_detected=1
fi
done
if [ $efa_detected -ne 1 ]; then
cat >&2 << EOF
An EFA device was not detected. Please verify that EFA has been enabled
for your Elastic Network Interface.
EOF
exit 1
fi
echo "EFA device detected: "
if command -v ibv_devices >/dev/null 2>&1; then
ibv_devices
fi
# check if GPUDirect RDMA is supported by
# reading from sysfs file "/sys/class/infiniband/<device_name>/gdr"
efa_gdr_enabled=0
for dev in /sys/class/infiniband/*/device; do
if [ "$(cat "${dev}"/gdr)" == "1" ]; then
echo "EFA GPUDirect RDMA support is enabled"
efa_gdr_enabled=1
else
echo "EFA GPUDirect RDMA support is not enabled"
fi
done
echo ""
echo "======== Configuration check ========"
# Check for memory lock limit and warn if less than 16GiB. 16GiB is enough for
# bounce buffers for 128 cores with some extra for safety.
if [ "$(ulimit -l)" != 'unlimited' ]; then
if [ "$(ulimit -l)" -lt "$((16*1024*1024))" ]; then
cat >&2 << EOF
Warning: EFA requires memory locking and the current limit may be too low for
your application.
EOF
ret=1
fi
fi
echo "Current memory lock limit: $(ulimit -l)"
huge_pages_size=$(grep "^Hugepagesize:" /proc/meminfo | awk '{print $2}')
huge_pages_file="/sys/kernel/mm/hugepages/hugepages-${huge_pages_size}kB/nr_hugepages"
hugepages=$(cat $huge_pages_file)
efa_ep_huge_pages_memory=$((110 * 1024)) # convert to kB
number_of_cores=$(lscpu | grep "^CPU(s):" | awk '{print $2}')
efa_total_huge_pages_memory=$(($efa_ep_huge_pages_memory * $number_of_cores))
efa_number_of_huge_pages=$(($efa_total_huge_pages_memory / $huge_pages_size + 1))
# For each end point, the libfabric EFA provider will create two packet pools,
# which is backed by huge page memory. The two packet pools will use 110 MB of
# memory. We need to reserve at least cores * 110 MB worth of memory in huge
# pages.
# RHEL 8 on ARM has a different huge page configuration than other
# OSes and defaulted to 512 MB huge page size. This might
# 1. massively over-reserve huge pages, thus run the machine effectively OOM
# 2. not reserve sufficient pages for a process per core worth of huge pages
# To deal with this issue, EFA-Config package sets a huge page size threshold as 16 MB (16384 KB).
# If the default huge page size is larger than the threshold, the huge page is not used.
huge_page_size_threshold=16384
if [[ $huge_pages_size -le $huge_page_size_threshold && $hugepages -lt $efa_number_of_huge_pages ]]; then
cat >&2 << EOF
Warning: Configuring huge pages is recommended for the best performance with
EFA.
EOF
ret=1
fi
echo "Current number of $huge_pages_size kB huge pages: $hugepages"
echo ""
echo "======== Software information ========"
echo "Kernel version: $(uname -r)"
# Verify that the EFA kernel driver and its dependencies are loaded.
if [ "$(grep -c -E '^ib_uverbs|^ib_core' /proc/modules)" -ne 2 ]; then
cat >&2 << EOF
Error: The ib_uverbs and ib_core kernel modules are required for the EFA kernel
module to be loaded.
EOF
exit 1
fi
echo "ib_uverbs and ib_core kernel modules are loaded"
if ! grep -q '^efa' /proc/modules; then
cat >&2 << EOF
Error: The EFA kernel module is not loaded. Please verify that the EFA kernel
module is provided with the kernel or is installed using DKMS.
EOF
exit 1
fi
if grep -q '^nvidia' /proc/modules; then
echo "NVIDIA kernel module is loaded, version: $(sudo modinfo -F version nvidia)"
else
echo "NVIDIA kernel module is not loaded"
fi
echo "EFA kernel module is loaded, version: $(sudo modinfo -F version efa)"
# Check for rdma-core and print the version.
libibverbs=$(sudo ldconfig -p | tr -d '\t' | grep '^libibverbs.so.1')
if [ $? -ne 0 ]; then
cat >&2 << EOF
Error: libibverbs shared library not found and is required for the EFA
libfabric provider.
EOF
exit 1
fi
echo "libibverbs in ldcache: $(readlink -m "$(echo "$libibverbs" | awk '{print $4}')")"
libefa=$(sudo ldconfig -p | tr -d '\t' | grep '^libefa.so.1')
if [ $? -ne 0 ]; then
cat >&2 << EOF
Error: libefa shared library not found and is required for the EFA
libfabric provider.
EOF
exit 1
fi
echo "libefa in ldcache: $(readlink -m "$(echo "$libefa" | awk '{print $4}')")"
if [ $skip_libfabric -eq 0 ]; then
if ! libfabric_checks; then
ret=1
fi
fi
if [ $skip_mpi -eq 0 ]; then
if ! mpi_checks; then
ret=1
fi
fi
echo ""
if [ $ret -ne 0 ]; then
echo "EFA check complete, please see output for warnings."
exit $ret
fi
echo "EFA check complete."
exit 0