blob: aac5c20bba4e463bfc114ea4c7b505383e4e98d4 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/usr/bin/env bash
MODE="${1}"
MY_PORT="${2}"
BACKEND_HOST="${3}"
BACKEND_PORT="${4}"
FILE="${NETDATA_NC_BACKEND_DIR-/tmp}/netdata-nc-backend-${MY_PORT}"
log() {
logger --stderr --id=$$ --tag "netdata-nc-backend" "${*}"
}
mync() {
local ret
log "Running: nc ${*}"
nc "${@}"
ret=$?
log "nc stopped with return code ${ret}."
return ${ret}
}
listen_save_replay_forever() {
local file="${1}" port="${2}" real_backend_host="${3}" real_backend_port="${4}" ret delay=1 started ended
while [ 1 ]
do
log "Starting nc to listen on port ${port} and save metrics to ${file}"
started=$(date +%s)
mync -l -p "${port}" | tee -a -p --output-error=exit "${file}"
ended=$(date +%s)
if [ -s "${file}" ]
then
if [ ! -z "${real_backend_host}" -a ! -z "${real_backend_port}" ]
then
log "Attempting to send the metrics to the real backend at ${real_backend_host}:${real_backend_port}"
mync "${real_backend_host}" "${real_backend_port}" <"${file}"
ret=$?
if [ ${ret} -eq 0 ]
then
log "Successfuly sent the metrics to ${real_backend_host}:${real_backend_port}"
mv "${file}" "${file}.old"
touch "${file}"
else
log "Failed to send the metrics to ${real_backend_host}:${real_backend_port} (nc returned ${ret}) - appending more data to ${file}"
fi
else
log "No backend configured - appending more data to ${file}"
fi
fi
# prevent a CPU hungry infinite loop
# if nc cannot listen to port
if [ $((ended - started)) -lt 5 ]
then
log "nc has been stopped too fast."
delay=30
else
delay=1
fi
log "Waiting ${delay} seconds before listening again for data."
sleep ${delay}
done
}
if [ "${MODE}" = "start" ]
then
# start the listener, in exclusive mode
# only one can use the same file/port at a time
{
flock -n 9
if [ $? -ne 0 ]
then
log "Cannot get exclusive lock on file ${FILE}.lock - Am I running multiple times?"
exit 2
fi
# save our PID to the lock file
echo "$$" >"${FILE}.lock"
listen_save_replay_forever "${FILE}" ${MY_PORT} ${BACKEND_HOST} ${BACKEND_PORT}
ret=$?
log "listener exited."
exit ${ret}
} 9>>"${FILE}.lock"
# we can only get here if ${FILE}.lock cannot be created
log "Cannot create file ${FILE}."
exit 3
elif [ "${MODE}" = "stop" ]
then
{
flock -n 9
if [ $? -ne 0 ]
then
pid=$(<${FILE}.lock)
log "Killing process ${pid}..."
kill -TERM -${pid}
exit 0
fi
log "File ${FILE}.lock has been locked by me but it shouldn't. Is a collector running?"
exit 4
} 9<"${FILE}.lock"
log "File ${FILE}.lock does not exist. Is a collector running?"
exit 5
else
cat <<EOF
Usage:
"${0}" start|stop PORT [BACKEND_HOST BACKEND_PORT]
PORT The port this script will listen
(configure netdata to use this as a second backend)
BACKEND_HOST The real backend host
BACKEND_PORT The real backend port
This script can act as fallback backend for netdata.
It will receive metrics from netdata, save them to
${FILE}
and once netdata reconnects to the real-backend, this script
will push all metrics collected to the real-backend too and
wait for a failure to happen again.
Only one netdata can connect to this script at a time.
If you need fallback for multiple netdata, run this script
multiple times with different ports.
You can run me in the background with this:
screen -d -m "${0}" start PORT [BACKEND_HOST BACKEND_PORT]
EOF
exit 1
fi
|
