Adding upstream version 1.1.0.upstream/1.1.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 500 insertions, 0 deletions

diff --git a/pkg/config/config.go b/pkg/config/config.go
new file mode 100644
index 0000000..1388ba9
--- /dev/null
+++ b/pkg/config/config.go
@@ -0,0 +1,135 @@
+package config
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"github.com/creasty/defaults"
+	"github.com/goccy/go-yaml"
+	"github.com/jessevdk/go-flags"
+	"github.com/pkg/errors"
+	"io/ioutil"
+	"os"
+)
+
+// Config defines Icinga DB config.
+type Config struct {
+	Database  Database  `yaml:"database"`
+	Redis     Redis     `yaml:"redis"`
+	Logging   Logging   `yaml:"logging"`
+	Retention Retention `yaml:"retention"`
+}
+
+// Validate checks constraints in the supplied configuration and returns an error if they are violated.
+func (c *Config) Validate() error {
+	if err := c.Database.Validate(); err != nil {
+		return err
+	}
+	if err := c.Redis.Validate(); err != nil {
+		return err
+	}
+	if err := c.Logging.Validate(); err != nil {
+		return err
+	}
+	if err := c.Retention.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Flags defines CLI flags.
+type Flags struct {
+	// Version decides whether to just print the version and exit.
+	Version bool `long:"version" description:"print version and exit"`
+	// Config is the path to the config file
+	Config string `short:"c" long:"config" description:"path to config file" required:"true" default:"/etc/icingadb/config.yml"`
+}
+
+// FromYAMLFile returns a new Config value created from the given YAML config file.
+func FromYAMLFile(name string) (*Config, error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't open YAML file "+name)
+	}
+	defer f.Close()
+
+	c := &Config{}
+	d := yaml.NewDecoder(f)
+
+	if err := defaults.Set(c); err != nil {
+		return nil, errors.Wrap(err, "can't set config defaults")
+	}
+
+	if err := d.Decode(c); err != nil {
+		return nil, errors.Wrap(err, "can't parse YAML file "+name)
+	}
+
+	if err := c.Validate(); err != nil {
+		return nil, errors.Wrap(err, "invalid configuration")
+	}
+
+	return c, nil
+}
+
+// ParseFlags parses CLI flags and
+// returns a Flags value created from them.
+func ParseFlags() (*Flags, error) {
+	f := &Flags{}
+	parser := flags.NewParser(f, flags.Default)
+
+	if _, err := parser.Parse(); err != nil {
+		return nil, errors.Wrap(err, "can't parse CLI flags")
+	}
+
+	return f, nil
+}
+
+// TLS provides TLS configuration options for Redis and Database.
+type TLS struct {
+	Enable   bool   `yaml:"tls"`
+	Cert     string `yaml:"cert"`
+	Key      string `yaml:"key"`
+	Ca       string `yaml:"ca"`
+	Insecure bool   `yaml:"insecure"`
+}
+
+// MakeConfig assembles a tls.Config from t and serverName.
+func (t *TLS) MakeConfig(serverName string) (*tls.Config, error) {
+	if !t.Enable {
+		return nil, nil
+	}
+
+	tlsConfig := &tls.Config{}
+	if t.Cert == "" {
+		if t.Key != "" {
+			return nil, errors.New("private key given, but client certificate missing")
+		}
+	} else if t.Key == "" {
+		return nil, errors.New("client certificate given, but private key missing")
+	} else {
+		crt, err := tls.LoadX509KeyPair(t.Cert, t.Key)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't load X.509 key pair")
+		}
+
+		tlsConfig.Certificates = []tls.Certificate{crt}
+	}
+
+	if t.Insecure {
+		tlsConfig.InsecureSkipVerify = true
+	} else if t.Ca != "" {
+		raw, err := ioutil.ReadFile(t.Ca)
+		if err != nil {
+			return nil, errors.Wrap(err, "can't read CA file")
+		}
+
+		tlsConfig.RootCAs = x509.NewCertPool()
+		if !tlsConfig.RootCAs.AppendCertsFromPEM(raw) {
+			return nil, errors.New("can't parse CA file")
+		}
+	}
+
+	tlsConfig.ServerName = serverName
+
+	return tlsConfig, nil
+}
diff --git a/pkg/config/database.go b/pkg/config/database.go
new file mode 100644
index 0000000..b42ff8e
--- /dev/null
+++ b/pkg/config/database.go
@@ -0,0 +1,175 @@
+package config
+
+import (
+	"fmt"
+	"github.com/go-sql-driver/mysql"
+	"github.com/icinga/icingadb/pkg/driver"
+	"github.com/icinga/icingadb/pkg/icingadb"
+	"github.com/icinga/icingadb/pkg/logging"
+	"github.com/icinga/icingadb/pkg/utils"
+	"github.com/jmoiron/sqlx"
+	"github.com/jmoiron/sqlx/reflectx"
+	"github.com/pkg/errors"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+var registerDriverOnce sync.Once
+
+// Database defines database client configuration.
+type Database struct {
+	Type       string           `yaml:"type" default:"mysql"`
+	Host       string           `yaml:"host"`
+	Port       int              `yaml:"port"`
+	Database   string           `yaml:"database"`
+	User       string           `yaml:"user"`
+	Password   string           `yaml:"password"`
+	TlsOptions TLS              `yaml:",inline"`
+	Options    icingadb.Options `yaml:"options"`
+}
+
+// Open prepares the DSN string and driver configuration,
+// calls sqlx.Open, but returns *icingadb.DB.
+func (d *Database) Open(logger *logging.Logger) (*icingadb.DB, error) {
+	registerDriverOnce.Do(func() {
+		driver.Register(logger)
+	})
+
+	var dsn string
+	switch d.Type {
+	case "mysql":
+		config := mysql.NewConfig()
+
+		config.User = d.User
+		config.Passwd = d.Password
+
+		if d.isUnixAddr() {
+			config.Net = "unix"
+			config.Addr = d.Host
+		} else {
+			config.Net = "tcp"
+			port := d.Port
+			if port == 0 {
+				port = 3306
+			}
+			config.Addr = net.JoinHostPort(d.Host, fmt.Sprint(port))
+		}
+
+		config.DBName = d.Database
+		config.Timeout = time.Minute
+		config.Params = map[string]string{"sql_mode": "ANSI_QUOTES"}
+
+		tlsConfig, err := d.TlsOptions.MakeConfig(d.Host)
+		if err != nil {
+			return nil, err
+		}
+
+		if tlsConfig != nil {
+			config.TLSConfig = "icingadb"
+			if err := mysql.RegisterTLSConfig(config.TLSConfig, tlsConfig); err != nil {
+				return nil, errors.Wrap(err, "can't register TLS config")
+			}
+		}
+
+		dsn = config.FormatDSN()
+	case "pgsql":
+		uri := &url.URL{
+			Scheme: "postgres",
+			User:   url.UserPassword(d.User, d.Password),
+			Path:   "/" + url.PathEscape(d.Database),
+		}
+
+		query := url.Values{
+			"connect_timeout":   {"60"},
+			"binary_parameters": {"yes"},
+
+			// Host and port can alternatively be specified in the query string. lib/pq can't parse the connection URI
+			// if a Unix domain socket path is specified in the host part of the URI, therefore always use the query
+			// string. See also https://github.com/lib/pq/issues/796
+			"host": {d.Host},
+		}
+		if d.Port != 0 {
+			query["port"] = []string{strconv.FormatInt(int64(d.Port), 10)}
+		}
+
+		if _, err := d.TlsOptions.MakeConfig(d.Host); err != nil {
+			return nil, err
+		}
+
+		if d.TlsOptions.Enable {
+			if d.TlsOptions.Insecure {
+				query["sslmode"] = []string{"require"}
+			} else {
+				query["sslmode"] = []string{"verify-full"}
+			}
+
+			if d.TlsOptions.Cert != "" {
+				query["sslcert"] = []string{d.TlsOptions.Cert}
+			}
+
+			if d.TlsOptions.Key != "" {
+				query["sslkey"] = []string{d.TlsOptions.Key}
+			}
+
+			if d.TlsOptions.Ca != "" {
+				query["sslrootcert"] = []string{d.TlsOptions.Ca}
+			}
+		} else {
+			query["sslmode"] = []string{"disable"}
+		}
+
+		uri.RawQuery = query.Encode()
+		dsn = uri.String()
+	default:
+		return nil, unknownDbType(d.Type)
+	}
+
+	db, err := sqlx.Open("icingadb-"+d.Type, dsn)
+	if err != nil {
+		return nil, errors.Wrap(err, "can't open database")
+	}
+
+	db.SetMaxIdleConns(d.Options.MaxConnections / 3)
+	db.SetMaxOpenConns(d.Options.MaxConnections)
+
+	db.Mapper = reflectx.NewMapperFunc("db", func(s string) string {
+		return utils.Key(s, '_')
+	})
+
+	return icingadb.NewDb(db, logger, &d.Options), nil
+}
+
+// Validate checks constraints in the supplied database configuration and returns an error if they are violated.
+func (d *Database) Validate() error {
+	switch d.Type {
+	case "mysql", "pgsql":
+	default:
+		return unknownDbType(d.Type)
+	}
+
+	if d.Host == "" {
+		return errors.New("database host missing")
+	}
+
+	if d.User == "" {
+		return errors.New("database user missing")
+	}
+
+	if d.Database == "" {
+		return errors.New("database name missing")
+	}
+
+	return d.Options.Validate()
+}
+
+func (d *Database) isUnixAddr() bool {
+	return strings.HasPrefix(d.Host, "/")
+}
+
+func unknownDbType(t string) error {
+	return errors.Errorf(`unknown database type %q, must be one of: "mysql", "pgsql"`, t)
+}
diff --git a/pkg/config/history_retention.go b/pkg/config/history_retention.go
new file mode 100644
index 0000000..d4373b7
--- /dev/null
+++ b/pkg/config/history_retention.go
@@ -0,0 +1,30 @@
+package config
+
+import (
+	"github.com/icinga/icingadb/pkg/icingadb/history"
+	"github.com/pkg/errors"
+	"time"
+)
+
+// Retention defines configuration for history retention.
+type Retention struct {
+	HistoryDays uint64                   `yaml:"history-days"`
+	SlaDays     uint64                   `yaml:"sla-days"`
+	Interval    time.Duration            `yaml:"interval" default:"1h"`
+	Count       uint64                   `yaml:"count" default:"5000"`
+	Options     history.RetentionOptions `yaml:"options"`
+}
+
+// Validate checks constraints in the supplied retention configuration and
+// returns an error if they are violated.
+func (r *Retention) Validate() error {
+	if r.Interval <= 0 {
+		return errors.New("retention interval must be positive")
+	}
+
+	if r.Count == 0 {
+		return errors.New("count must be greater than zero")
+	}
+
+	return r.Options.Validate()
+}
diff --git a/pkg/config/logging.go b/pkg/config/logging.go
new file mode 100644
index 0000000..9ccd35e
--- /dev/null
+++ b/pkg/config/logging.go
@@ -0,0 +1,44 @@
+package config
+
+import (
+	"github.com/icinga/icingadb/pkg/logging"
+	"github.com/pkg/errors"
+	"go.uber.org/zap/zapcore"
+	"os"
+	"time"
+)
+
+// Logging defines Logger configuration.
+type Logging struct {
+	// zapcore.Level at 0 is for info level.
+	Level  zapcore.Level `yaml:"level" default:"0"`
+	Output string        `yaml:"output"`
+	// Interval for periodic logging.
+	Interval time.Duration `yaml:"interval" default:"20s"`
+
+	logging.Options `yaml:"options"`
+}
+
+// Validate checks constraints in the supplied Logging configuration and returns an error if they are violated.
+// Also configures the log output if it is not configured:
+// systemd-journald is used when Icinga DB is running under systemd, otherwise stderr.
+func (l *Logging) Validate() error {
+	if l.Interval <= 0 {
+		return errors.New("periodic logging interval must be positive")
+	}
+
+	if l.Output == "" {
+		if _, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
+			// When started by systemd, NOTIFY_SOCKET is set by systemd for Type=notify supervised services,
+			// which is the default setting for the Icinga DB service.
+			// This assumes that Icinga DB is running under systemd, so set output to systemd-journald.
+			l.Output = logging.JOURNAL
+		} else {
+			// Otherwise set it to console, i.e. write log messages to stderr.
+			l.Output = logging.CONSOLE
+		}
+	}
+
+	// To be on the safe side, always call AssertOutput.
+	return logging.AssertOutput(l.Output)
+}
diff --git a/pkg/config/redis.go b/pkg/config/redis.go
new file mode 100644
index 0000000..38571e3
--- /dev/null
+++ b/pkg/config/redis.go
@@ -0,0 +1,116 @@
+package config
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	"github.com/icinga/icingadb/pkg/backoff"
+	"github.com/icinga/icingadb/pkg/icingaredis"
+	"github.com/icinga/icingadb/pkg/logging"
+	"github.com/icinga/icingadb/pkg/retry"
+	"github.com/icinga/icingadb/pkg/utils"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"net"
+	"strings"
+	"time"
+)
+
+// Redis defines Redis client configuration.
+type Redis struct {
+	Host       string              `yaml:"host"`
+	Port       int                 `yaml:"port" default:"6380"`
+	Password   string              `yaml:"password"`
+	TlsOptions TLS                 `yaml:",inline"`
+	Options    icingaredis.Options `yaml:"options"`
+}
+
+type ctxDialerFunc = func(ctx context.Context, network, addr string) (net.Conn, error)
+
+// NewClient prepares Redis client configuration,
+// calls redis.NewClient, but returns *icingaredis.Client.
+func (r *Redis) NewClient(logger *logging.Logger) (*icingaredis.Client, error) {
+	tlsConfig, err := r.TlsOptions.MakeConfig(r.Host)
+	if err != nil {
+		return nil, err
+	}
+
+	var dialer ctxDialerFunc
+	dl := &net.Dialer{Timeout: 15 * time.Second}
+
+	if tlsConfig == nil {
+		dialer = dl.DialContext
+	} else {
+		dialer = (&tls.Dialer{NetDialer: dl, Config: tlsConfig}).DialContext
+	}
+
+	options := &redis.Options{
+		Dialer:      dialWithLogging(dialer, logger),
+		Password:    r.Password,
+		DB:          0, // Use default DB,
+		ReadTimeout: r.Options.Timeout,
+		TLSConfig:   tlsConfig,
+	}
+
+	if strings.HasPrefix(r.Host, "/") {
+		options.Network = "unix"
+		options.Addr = r.Host
+	} else {
+		options.Network = "tcp"
+		options.Addr = net.JoinHostPort(r.Host, fmt.Sprint(r.Port))
+	}
+
+	c := redis.NewClient(options)
+
+	opts := c.Options()
+	opts.PoolSize = utils.MaxInt(32, opts.PoolSize)
+	opts.MaxRetries = opts.PoolSize + 1 // https://github.com/go-redis/redis/issues/1737
+	c = redis.NewClient(opts)
+
+	return icingaredis.NewClient(c, logger, &r.Options), nil
+}
+
+// dialWithLogging returns a Redis Dialer with logging capabilities.
+func dialWithLogging(dialer ctxDialerFunc, logger *logging.Logger) ctxDialerFunc {
+	// dial behaves like net.Dialer#DialContext,
+	// but re-tries on common errors that are considered retryable.
+	return func(ctx context.Context, network, addr string) (conn net.Conn, err error) {
+		err = retry.WithBackoff(
+			ctx,
+			func(ctx context.Context) (err error) {
+				conn, err = dialer(ctx, network, addr)
+				return
+			},
+			retry.Retryable,
+			backoff.NewExponentialWithJitter(1*time.Millisecond, 1*time.Second),
+			retry.Settings{
+				Timeout: 5 * time.Minute,
+				OnError: func(_ time.Duration, _ uint64, err, lastErr error) {
+					if lastErr == nil || err.Error() != lastErr.Error() {
+						logger.Warnw("Can't connect to Redis. Retrying", zap.Error(err))
+					}
+				},
+				OnSuccess: func(elapsed time.Duration, attempt uint64, _ error) {
+					if attempt > 0 {
+						logger.Infow("Reconnected to Redis",
+							zap.Duration("after", elapsed), zap.Uint64("attempts", attempt+1))
+					}
+				},
+			},
+		)
+
+		err = errors.Wrap(err, "can't connect to Redis")
+
+		return
+	}
+}
+
+// Validate checks constraints in the supplied Redis configuration and returns an error if they are violated.
+func (r *Redis) Validate() error {
+	if r.Host == "" {
+		return errors.New("Redis host missing")
+	}
+
+	return r.Options.Validate()
+}