1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
package driver
import (
"context"
"database/sql"
"database/sql/driver"
"github.com/go-sql-driver/mysql"
"github.com/icinga/icingadb/pkg/backoff"
"github.com/icinga/icingadb/pkg/icingaredis/telemetry"
"github.com/icinga/icingadb/pkg/logging"
"github.com/icinga/icingadb/pkg/retry"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"go.uber.org/zap"
"time"
)
const MySQL = "icingadb-mysql"
const PostgreSQL = "icingadb-pgsql"
var timeout = time.Minute * 5
// RetryConnector wraps driver.Connector with retry logic.
type RetryConnector struct {
driver.Connector
driver Driver
}
// Connect implements part of the driver.Connector interface.
func (c RetryConnector) Connect(ctx context.Context) (driver.Conn, error) {
var conn driver.Conn
err := errors.Wrap(retry.WithBackoff(
ctx,
func(ctx context.Context) (err error) {
conn, err = c.Connector.Connect(ctx)
return
},
shouldRetry,
backoff.NewExponentialWithJitter(time.Millisecond*128, time.Minute*1),
retry.Settings{
Timeout: timeout,
OnError: func(_ time.Duration, _ uint64, err, lastErr error) {
telemetry.UpdateCurrentDbConnErr(err)
if lastErr == nil || err.Error() != lastErr.Error() {
c.driver.Logger.Warnw("Can't connect to database. Retrying", zap.Error(err))
}
},
OnSuccess: func(elapsed time.Duration, attempt uint64, _ error) {
telemetry.UpdateCurrentDbConnErr(nil)
if attempt > 0 {
c.driver.Logger.Infow("Reconnected to database",
zap.Duration("after", elapsed), zap.Uint64("attempts", attempt+1))
}
},
},
), "can't connect to database")
return conn, err
}
// Driver implements part of the driver.Connector interface.
func (c RetryConnector) Driver() driver.Driver {
return c.driver
}
// Driver wraps a driver.Driver that also must implement driver.DriverContext with logging capabilities and provides our RetryConnector.
type Driver struct {
ctxDriver
Logger *logging.Logger
}
// OpenConnector implements the DriverContext interface.
func (d Driver) OpenConnector(name string) (driver.Connector, error) {
c, err := d.ctxDriver.OpenConnector(name)
if err != nil {
return nil, err
}
return &RetryConnector{
driver: d,
Connector: c,
}, nil
}
// Register makes our database Driver available under the name "icingadb-*sql".
func Register(logger *logging.Logger) {
sql.Register(MySQL, &Driver{ctxDriver: &mysql.MySQLDriver{}, Logger: logger})
sql.Register(PostgreSQL, &Driver{ctxDriver: PgSQLDriver{}, Logger: logger})
_ = mysql.SetLogger(mysqlLogger(func(v ...interface{}) { logger.Debug(v...) }))
sqlx.BindDriver(PostgreSQL, sqlx.DOLLAR)
}
// ctxDriver helps ensure that we only support drivers that implement driver.Driver and driver.DriverContext.
type ctxDriver interface {
driver.Driver
driver.DriverContext
}
// mysqlLogger is an adapter that allows ordinary functions to be used as a logger for mysql.SetLogger.
type mysqlLogger func(v ...interface{})
// Print implements the mysql.Logger interface.
func (log mysqlLogger) Print(v ...interface{}) {
log(v)
}
func shouldRetry(err error) bool {
if errors.Is(err, driver.ErrBadConn) {
return true
}
return retry.Retryable(err)
}
|
