Merging upstream version 1.16.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 142 insertions, 31 deletions

diff --git a/daemon/anonymous-statistics.sh.in b/daemon/anonymous-statistics.sh.in
index 9f548ce96..7d73f6d61 100755
--- a/daemon/anonymous-statistics.sh.in
+++ b/daemon/anonymous-statistics.sh.in
@@ -22,6 +22,9 @@ if [ -f "@configdir_POST@/.opt-out-from-anonymous-statistics" ]; then
 	exit 0
 fi
 
+# Shorten version for easier reporting
+NETDATA_VERSION=$(echo "${NETDATA_VERSION}" | sed 's/-.*//g' | tr -d 'v')
+
 echo "&av=${NETDATA_VERSION}\
 &ec=${ACTION}\
 &ea=${ACTION_RESULT}\
@@ -62,7 +65,7 @@ if [ -n "$(command -v curl 2>/dev/null)" ]; then
 		--data-urlencode "cd2=${NETDATA_SYSTEM_OS_ID}" \
 		--data-urlencode "cd3=${NETDATA_SYSTEM_OS_ID_LIKE}" \
 		--data-urlencode "cd4=${NETDATA_SYSTEM_OS_VERSION}" \
-		--data-urlencode "cd5=${NETDATA_SYSTEM_OS_DETECTION}" \
+		--data-urlencode "cd5=${NETDATA_SYSTEM_OS_VERSION_ID}" \
 		--data-urlencode "cd6=${NETDATA_SYSTEM_OS_DETECTION}" \
 		--data-urlencode "cd7=${NETDATA_SYSTEM_KERNEL_NAME}" \
 		--data-urlencode "cd8=${NETDATA_SYSTEM_KERNEL_VERSION}" \
diff --git a/daemon/common.h b/daemon/common.h
index 9a55fa3af..a15ddb877 100644
--- a/daemon/common.h
+++ b/daemon/common.h
@@ -14,6 +14,7 @@
 #define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value)
 #define config_get_boolean(section, name, value) appconfig_get_boolean(&netdata_config, section, name, value)
 #define config_get_boolean_ondemand(section, name, value) appconfig_get_boolean_ondemand(&netdata_config, section, name, value)
+#define config_get_duration(section, name, value) appconfig_get_duration(&netdata_config, section, name, value)
 
 #define config_set(section, name, default_value) appconfig_set(&netdata_config, section, name, default_value)
 #define config_set_default(section, name, value) appconfig_set_default(&netdata_config, section, name, value)
diff --git a/daemon/config/README.md b/daemon/config/README.md
index 0508a19d9..4778cad24 100644
--- a/daemon/config/README.md
+++ b/daemon/config/README.md
@@ -62,7 +62,7 @@ memory deduplication (ksm) | `yes` | When set to `yes`, netdata will offer its i
 TZ environment variable | `:/etc/localtime` | Where to find the timezone
 timezone | auto-detected | The timezone retrieved from the environment variable
 debug flags | `0x0000000000000000` | Bitmap of debug options to enable. For more information check [Tracing Options](../#debugging).
-debug log | `/var/log/netdata/debug.log` | The filename to save debug information. This file will not be created is debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `none` to disable this log. For more information check [Tracing Options](../#debugging).
+debug log | `/var/log/netdata/debug.log` | The filename to save debug information. This file will not be created if debugging is not enabled. You can also set it to `syslog` to send the debug messages to syslog, or `none` to disable this log. For more information check [Tracing Options](../#debugging).
 error log | `/var/log/netdata/error.log` | The filename to save error messages for netdata daemon and all plugins (`stderr` is sent here for all netdata programs, including the plugins). You can also set it to `syslog` to send the errors to syslog, or `none` to disable this log.
 access log | `/var/log/netdata/access.log` | The filename to save the log of web clients accessing netdata charts. You can also set it to `syslog` to send the access log to syslog, or `none` to disable this log.
 errors flood protection period | `1200` | UNUSED - Length of period (in sec) during which the number of errors should not exceed the `errors to trigger flood protection`.
@@ -105,12 +105,12 @@ setting | default | info
 :------:|:-------:|:----
 enabled | `yes` | Set to `no` to disable all alarms and notifications
 in memory max health log entries | 1000 | Size of the alarm history held in RAM
-script to execute on alarm | `/usr/libexec/netdata/plugins.d/alarm-notify.sh` | The script that sends alarm notifications.
+script to execute on alarm | `/usr/libexec/netdata/plugins.d/alarm-notify.sh` | The script that sends alarm notifications. Note that in versions before 1.16, the plugins.d directory may be installed in a different location in certain OSs (e.g. under `/usr/lib/netdata`).
 stock health configuration directory | `/usr/lib/netdata/conf.d/health.d` | Contains the stock alarm configuration files for each collector
 health configuration directory | `/etc/netdata/health.d` | The directory containing the user alarm configuration files, to override the stock configurations
 run at least every seconds | `10` | Controls how often all alarm conditions should be evaluated.
 postpone alarms during hibernation for seconds | `60` | Prevents false alarms. May need to be increased if you get alarms during hibernation.
-rotate log every lines | 2000 | Controls the number of alarm log entries stored in `<lib directory>/health-log.db`, where <lib directory> is the one configured in the [[global] section](#global-section-options)
+rotate log every lines | 2000 | Controls the number of alarm log entries stored in `<lib directory>/health-log.db`, where `<lib directory>` is the one configured in the [[global] section](#global-section-options)
 
 ### [registry] section options
 
@@ -128,6 +128,8 @@ The configuration options for plugins appear in sections following the pattern `
 
 Most internal plugins will provide additional options. Check [Internal Plugins](../../collectors/) for more information.
 
+Please note, that by default Netdata will enable monitoring metrics for disks, memory, and network only when they are not zero. If they are constantly zero they are ignored. Metrics that will start having values, after netdata is started, will be detected and charts will be automatically added to the dashboard (a refresh of the dashboard is needed for them to appear though). Use `yes` instead of `auto` in plugin configuration sections to enable these charts permanently.
+
 #### External plugins
 
 External plugins will have only 2 options at `netdata.conf`:
@@ -141,6 +143,6 @@ External plugins that need additional configuration may support a dedicated file
 
 ### Per chart configuration
 
-In this section you will a separate subsection for each chart shown on the dashboard. You can control all aspects of a specific chart here. You can understand what each option does by reading [how charts are defined](../../collectors/plugins.d/#chart). If you don't know how to find the name of a chart, you can learn about it [here](../../docs/Charts.md).
+In this section you will find a separate subsection for each chart shown on the dashboard. You can control all aspects of a specific chart here. You can understand what each option does by reading [how charts are defined](../../collectors/plugins.d/#chart). If you don't know how to find the name of a chart, you can learn about it [here](../../docs/Charts.md).
 
 [![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fdaemon%2Fconfig%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 9cc05abb7..53b7546f2 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -535,10 +535,10 @@ void global_statistics_charts(void) {
 
 #ifdef ENABLE_DBENGINE
     if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        unsigned long long stats_array[27];
+        unsigned long long stats_array[RRDENG_NR_STATS];
 
         /* get localhost's DB engine's statistics */
-        rrdeng_get_27_statistics(localhost->rrdeng_ctx, stats_array);
+        rrdeng_get_33_statistics(localhost->rrdeng_ctx, stats_array);
 
         // ----------------------------------------------------------------
 
@@ -637,6 +637,7 @@ void global_statistics_charts(void) {
 
         {
             static RRDSET *st_pg_cache_pages = NULL;
+            static RRDDIM *rd_descriptors = NULL;
             static RRDDIM *rd_populated = NULL;
             static RRDDIM *rd_commited = NULL;
             static RRDDIM *rd_insertions = NULL;
@@ -660,6 +661,7 @@ void global_statistics_charts(void) {
                         , RRDSET_TYPE_LINE
                 );
 
+                rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
                 rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
                 rd_commited = rrddim_add(st_pg_cache_pages, "commited", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
                 rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -670,6 +672,7 @@ void global_statistics_charts(void) {
             else
                 rrdset_next(st_pg_cache_pages);
 
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]);
             rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
             rrddim_set_by_pointer(st_pg_cache_pages, rd_commited, (collected_number)stats_array[4]);
             rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]);
@@ -746,6 +749,75 @@ void global_statistics_charts(void) {
             rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]);
             rrdset_done(st_io_stats);
         }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_errors = NULL;
+            static RRDDIM *rd_fs_errors = NULL;
+            static RRDDIM *rd_io_errors = NULL;
+
+            if (unlikely(!st_errors)) {
+                st_errors = rrdset_create_localhost(
+                        "netdata"
+                        , "dbengine_global_errors"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine errors"
+                        , "errors/s"
+                        , "netdata"
+                        , "stats"
+                        , 130507
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_io_errors = rrddim_add(st_errors, "I/O errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+                rd_fs_errors = rrddim_add(st_errors, "FS errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+            }
+            else
+                rrdset_next(st_errors);
+
+            rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
+            rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
+            rrdset_done(st_errors);
+        }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_fd = NULL;
+            static RRDDIM *rd_fd_current = NULL;
+            static RRDDIM *rd_fd_max = NULL;
+
+            if (unlikely(!st_fd)) {
+                st_fd = rrdset_create_localhost(
+                        "netdata"
+                        , "dbengine_global_file_descriptors"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine File Descriptors"
+                        , "descriptors"
+                        , "netdata"
+                        , "stats"
+                        , 130508
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+                rd_fd_max = rrddim_add(st_fd, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            }
+            else
+                rrdset_next(st_fd);
+
+            rrddim_set_by_pointer(st_fd, rd_fd_current, (collected_number)stats_array[32]);
+            /* Careful here, modify this accordingly if the File-Descriptor budget ever changes */
+            rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4);
+            rrdset_done(st_fd);
+        }
     }
 #endif
 
diff --git a/daemon/main.c b/daemon/main.c
index a1577fb93..0ced90810 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -24,7 +24,7 @@ void netdata_cleanup_and_exit(int ret) {
     error_log_limit_unlimited();
     info("EXIT: netdata prepares to exit with code %d...", ret);
 
-	send_statistics("EXIT", ret?"ERROR":"OK","-");
+    send_statistics("EXIT", ret?"ERROR":"OK","-");
 
     // cleanup/save the database and exit
     info("EXIT: cleaning up the database...");
@@ -49,6 +49,10 @@ void netdata_cleanup_and_exit(int ret) {
             error("EXIT: cannot unlink pidfile '%s'.", pidfile);
     }
 
+#ifdef ENABLE_HTTPS
+    security_clean_openssl();
+#endif
+
     info("EXIT: all done - netdata is now exiting - bye bye...");
     exit(ret);
 }
@@ -345,7 +349,20 @@ static const char *verify_required_directory(const char *dir) {
     return dir;
 }
 
-void log_init(void) {
+#ifdef ENABLE_HTTPS
+static void security_init(){
+    char filename[FILENAME_MAX + 1];
+    snprintfz(filename, FILENAME_MAX, "%s/ssl/key.pem",netdata_configured_user_config_dir);
+    security_key    = config_get(CONFIG_SECTION_WEB, "ssl key",  filename);
+
+    snprintfz(filename, FILENAME_MAX, "%s/ssl/cert.pem",netdata_configured_user_config_dir);
+    security_cert    = config_get(CONFIG_SECTION_WEB, "ssl certificate",  filename);
+
+    security_openssl_library();
+}
+#endif
+
+static void log_init(void) {
     char filename[FILENAME_MAX + 1];
     snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir);
     stdout_filename    = config_get(CONFIG_SECTION_GLOBAL, "debug log",  filename);
@@ -356,9 +373,9 @@ void log_init(void) {
     snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir);
     stdaccess_filename = config_get(CONFIG_SECTION_GLOBAL, "access log", filename);
 
-	char deffacility[8];
-	snprintfz(deffacility,7,"%s","daemon");
-	facility_log = config_get(CONFIG_SECTION_GLOBAL, "facility log",  deffacility);
+    char deffacility[8];
+    snprintfz(deffacility,7,"%s","daemon");
+    facility_log = config_get(CONFIG_SECTION_GLOBAL, "facility log",  deffacility);
 
     error_log_throttle_period = config_get_number(CONFIG_SECTION_GLOBAL, "errors flood protection period", error_log_throttle_period);
     error_log_errors_per_period = (unsigned long)config_get_number(CONFIG_SECTION_GLOBAL, "errors to trigger flood protection", (long long int)error_log_errors_per_period);
@@ -420,8 +437,9 @@ static void get_netdata_configured_variables() {
     // get the hostname
 
     char buf[HOSTNAME_MAX + 1];
-    if(gethostname(buf, HOSTNAME_MAX) == -1)
+    if(gethostname(buf, HOSTNAME_MAX) == -1){
         error("Cannot get machine hostname.");
+    }
 
     netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf);
     debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname);
@@ -724,20 +742,20 @@ void send_statistics( const char *action, const char *action_result, const char
         if (likely(access(optout_file, R_OK) != 0)) {
             as_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("anonymous-statistics.sh") + 2));
             sprintf(as_script, "%s/%s", netdata_configured_primary_plugins_dir, "anonymous-statistics.sh");
-			if (unlikely(access(as_script, R_OK) != 0)) {
-				netdata_anonymous_statistics_enabled=0;
-				info("Anonymous statistics script %s not found.",as_script);
-				freez(as_script);
-			} else {
-				netdata_anonymous_statistics_enabled=1;
-			}
-		} else {
+            if (unlikely(access(as_script, R_OK) != 0)) {
+               netdata_anonymous_statistics_enabled=0;
+               info("Anonymous statistics script %s not found.",as_script);
+               freez(as_script);
+            } else {
+               netdata_anonymous_statistics_enabled=1;
+            }
+        } else {
             netdata_anonymous_statistics_enabled = 0;
             as_script = NULL;
         }
         freez(optout_file);
     }
-	if(!netdata_anonymous_statistics_enabled) return;
+    if(!netdata_anonymous_statistics_enabled) return;
     if (!action) return;
     if (!action_result) action_result="";
     if (!action_data) action_data="";
@@ -756,6 +774,12 @@ void send_statistics( const char *action, const char *action_result, const char
     freez(command_to_run);
 }
 
+void set_silencers_filename() {
+    char filename[FILENAME_MAX + 1];
+    snprintfz(filename, FILENAME_MAX, "%s/health.silencers.json", netdata_configured_varlib_dir);
+    silencers_filename = config_get(CONFIG_SECTION_HEALTH, "silencers file", filename);
+}
+
 int main(int argc, char **argv) {
     int i;
     int config_loaded = 0;
@@ -881,12 +905,9 @@ int main(int argc, char **argv) {
                             return 0;
                         }
                         else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
-                            unsigned history_seconds;
-
                             optarg += strlen(createdataset_string);
-                            history_seconds = (unsigned )strtoull(optarg, NULL, 0);
-
 #ifdef ENABLE_DBENGINE
+                            unsigned history_seconds = (unsigned )strtoull(optarg, NULL, 0);
                             generate_dbengine_dataset(history_seconds);
 #endif
                             return 0;
@@ -1081,6 +1102,17 @@ int main(int argc, char **argv) {
         error_log_limit_unlimited();
 
         // --------------------------------------------------------------------
+        // get the certificate and start security
+#ifdef ENABLE_HTTPS
+        security_init();
+#endif
+
+        // --------------------------------------------------------------------
+        // This is the safest place to start the SILENCERS structure
+        set_silencers_filename();
+        health_initialize_global_silencers();
+
+        // --------------------------------------------------------------------
         // setup process signals
 
         // block signals while initializing threads.
@@ -1134,10 +1166,6 @@ int main(int argc, char **argv) {
     // initialize the log files
     open_all_log_files();
 
-	netdata_anonymous_statistics_enabled=-1;
-    struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
-    if (get_system_info(system_info) == 0) send_statistics("START","-", "-");
-
 #ifdef NETDATA_INTERNAL_CHECKS
     if(debug_flags != 0) {
         struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY };
@@ -1171,8 +1199,11 @@ int main(int argc, char **argv) {
     // ------------------------------------------------------------------------
     // initialize rrd, registry, health, rrdpush, etc.
 
+    netdata_anonymous_statistics_enabled=-1;
+    struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
+    get_system_info(system_info);
+
     rrd_init(netdata_configured_hostname, system_info);
-    rrdhost_system_info_free(system_info);
     // ------------------------------------------------------------------------
     // enable log flood protection
 
@@ -1196,6 +1227,8 @@ int main(int argc, char **argv) {
 
     info("netdata initialization completed. Enjoy real-time performance monitoring!");
     netdata_ready = 1;
+  
+    send_statistics("START", "-",  "-");
 
     // ------------------------------------------------------------------------
     // unblock signals