Merging upstream version 1.34.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 367 insertions, 83 deletions

diff --git a/collectors/cgroups.plugin/README.md b/collectors/cgroups.plugin/README.md
index 86776d6e0..d74ef000e 100644
--- a/collectors/cgroups.plugin/README.md
+++ b/collectors/cgroups.plugin/README.md
@@ -284,4 +284,4 @@ So, when a network interface or container stops, Netdata might log a few errors
   
 Network interfaces are monitored by means of the [proc plugin](/collectors/proc.plugin/README.md#monitored-network-interface-metrics).
 
-[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fcollectors%2Fcgroups.plugin%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
+
diff --git a/collectors/cgroups.plugin/sys_fs_cgroup.c b/collectors/cgroups.plugin/sys_fs_cgroup.c
index 92aa22c77..8efb68cf0 100644
--- a/collectors/cgroups.plugin/sys_fs_cgroup.c
+++ b/collectors/cgroups.plugin/sys_fs_cgroup.c
@@ -13,6 +13,7 @@ static long system_page_size = 4096; // system will be queried via sysconf() in
 
 static int cgroup_enable_cpuacct_stat = CONFIG_BOOLEAN_AUTO;
 static int cgroup_enable_cpuacct_usage = CONFIG_BOOLEAN_AUTO;
+static int cgroup_enable_cpuacct_cpu_throttling = CONFIG_BOOLEAN_YES;
 static int cgroup_enable_memory = CONFIG_BOOLEAN_AUTO;
 static int cgroup_enable_detailed_memory = CONFIG_BOOLEAN_AUTO;
 static int cgroup_enable_memory_failcnt = CONFIG_BOOLEAN_AUTO;
@@ -72,6 +73,12 @@ static uint32_t Read_hash = 0;
 static uint32_t Write_hash = 0;
 static uint32_t user_hash = 0;
 static uint32_t system_hash = 0;
+static uint32_t user_usec_hash = 0;
+static uint32_t system_usec_hash = 0;
+static uint32_t nr_periods_hash = 0;
+static uint32_t nr_throttled_hash = 0;
+static uint32_t throttled_time_hash = 0;
+static uint32_t throttled_usec_hash = 0;
 
 enum cgroups_type { CGROUPS_AUTODETECT_FAIL, CGROUPS_V1, CGROUPS_V2 };
 
@@ -116,21 +123,41 @@ static enum cgroups_systemd_setting cgroups_detect_systemd(const char *exec)
     if (!f)
         return retval;
 
-    while (fgets(buf, MAXSIZE_PROC_CMDLINE, f) != NULL) {
-        if ((begin = strstr(buf, SYSTEMD_HIERARCHY_STRING))) {
-            end = begin = begin + strlen(SYSTEMD_HIERARCHY_STRING);
-            if (!*begin)
-                break;
-            while (isalpha(*end))
-                end++;
-            *end = 0;
-            for (int i = 0; cgroups_systemd_options[i].name; i++) {
-                if (!strcmp(begin, cgroups_systemd_options[i].name)) {
-                    retval = cgroups_systemd_options[i].setting;
+    fd_set rfds;
+    struct timeval timeout;
+    int fd = fileno(f);
+    int ret = -1;
+
+    FD_ZERO(&rfds);
+    FD_SET(fd, &rfds);
+    timeout.tv_sec = 3;
+    timeout.tv_usec = 0;
+
+    if (fd != -1) {
+        ret = select(fd + 1, &rfds, NULL, NULL, &timeout);
+    }
+
+    if (ret == -1) {
+        error("Failed to get the output of \"%s\"", exec);
+    } else if (ret == 0) {
+        info("Cannot get the output of \"%s\" within %"PRId64" seconds", exec, (int64_t)timeout.tv_sec);
+    } else {
+        while (fgets(buf, MAXSIZE_PROC_CMDLINE, f) != NULL) {
+            if ((begin = strstr(buf, SYSTEMD_HIERARCHY_STRING))) {
+                end = begin = begin + strlen(SYSTEMD_HIERARCHY_STRING);
+                if (!*begin)
                     break;
+                while (isalpha(*end))
+                    end++;
+                *end = 0;
+                for (int i = 0; cgroups_systemd_options[i].name; i++) {
+                    if (!strcmp(begin, cgroups_systemd_options[i].name)) {
+                        retval = cgroups_systemd_options[i].setting;
+                        break;
+                    }
                 }
+                break;
             }
-            break;
         }
     }
 
@@ -213,6 +240,14 @@ static enum cgroups_type cgroups_try_detect_version()
     return CGROUPS_V2;
 }
 
+void set_cgroup_base_path(char *filename, char *path) {
+    if (strncmp(netdata_configured_host_prefix, path, strlen(netdata_configured_host_prefix)) == 0) {
+        snprintfz(filename, FILENAME_MAX, "%s", path);
+    } else {
+        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, path);
+    }
+}
+
 void read_cgroup_plugin_configuration() {
     system_page_size = sysconf(_SC_PAGESIZE);
 
@@ -220,6 +255,12 @@ void read_cgroup_plugin_configuration() {
     Write_hash = simple_hash("Write");
     user_hash = simple_hash("user");
     system_hash = simple_hash("system");
+    user_usec_hash = simple_hash("user_usec");
+    system_usec_hash = simple_hash("system_usec");
+    nr_periods_hash = simple_hash("nr_periods");
+    nr_throttled_hash = simple_hash("nr_throttled");
+    throttled_time_hash = simple_hash("throttled_time");
+    throttled_usec_hash = simple_hash("throttled_usec");
 
     cgroup_update_every = (int)config_get_number("plugin:cgroups", "update every", localhost->rrd_update_every);
     if(cgroup_update_every < localhost->rrd_update_every)
@@ -241,6 +282,7 @@ void read_cgroup_plugin_configuration() {
 
     cgroup_enable_cpuacct_stat = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct stat (total CPU)", cgroup_enable_cpuacct_stat);
     cgroup_enable_cpuacct_usage = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct usage (per core CPU)", cgroup_enable_cpuacct_usage);
+    cgroup_enable_cpuacct_cpu_throttling = config_get_boolean_ondemand("plugin:cgroups", "enable cpuacct cpu throttling", cgroup_enable_cpuacct_cpu_throttling);
 
     cgroup_enable_memory = config_get_boolean_ondemand("plugin:cgroups", "enable memory", cgroup_enable_memory);
     cgroup_enable_detailed_memory = config_get_boolean_ondemand("plugin:cgroups", "enable detailed memory", cgroup_enable_detailed_memory);
@@ -285,7 +327,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup/cpuacct";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_cpuacct_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuacct", filename);
 
         mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "cpuset");
@@ -295,7 +337,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup/cpuset";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_cpuset_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/cpuset", filename);
 
         mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "blkio");
@@ -305,7 +347,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup/blkio";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_blkio_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/blkio", filename);
 
         mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "memory");
@@ -315,7 +357,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup/memory";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_memory_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/memory", filename);
 
         mi = mountinfo_find_by_filesystem_super_option(root, "cgroup", "devices");
@@ -325,7 +367,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup/devices";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_devices_base = config_get("plugin:cgroups", "path to /sys/fs/cgroup/devices", filename);
     }
     else {
@@ -357,7 +399,7 @@ void read_cgroup_plugin_configuration() {
             s = "/sys/fs/cgroup";
         }
         else s = mi->mount_point;
-        snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, s);
+        set_cgroup_base_path(filename, s);
         cgroup_unified_base = config_get("plugin:cgroups", "path to unified cgroups", filename);
         debug(D_CGROUP, "using cgroup root: '%s'", cgroup_unified_base);
     }
@@ -395,7 +437,7 @@ void read_cgroup_plugin_configuration() {
                     " !*.user "
                     " !/ "
                     " !/docker "
-                    " !/libvirt "
+                    " !*/libvirt "
                     " !/lxc "
                     " !/lxc/*/* "                          //  #1397 #2649
                     " !/lxc.monitor* "
@@ -630,12 +672,12 @@ struct memory {
 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
 struct cpuacct_stat {
     int updated;
-    int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+    int enabled;            // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
 
     char *filename;
 
-    unsigned long long user;
-    unsigned long long system;
+    unsigned long long user;           // v1, v2(user_usec)
+    unsigned long long system;         // v1, v2(system_usec)
 };
 
 // https://www.kernel.org/doc/Documentation/cgroup-v1/cpuacct.txt
@@ -649,6 +691,20 @@ struct cpuacct_usage {
     unsigned long long *cpu_percpu;
 };
 
+// represents cpuacct/cpu.stat, for v2 'cpuacct_stat' is used for 'user_usec', 'system_usec'
+struct cpuacct_cpu_throttling {
+    int updated;
+    int enabled; // CONFIG_BOOLEAN_YES or CONFIG_BOOLEAN_AUTO
+
+    char *filename;
+
+    unsigned long long nr_periods;
+    unsigned long long nr_throttled;
+    unsigned long long throttled_time;
+
+    unsigned long long nr_throttled_perc;
+};
+
 struct cgroup_network_interface {
     const char *host_device;
     const char *container_device;
@@ -677,6 +733,7 @@ struct cgroup {
 
     struct cpuacct_stat cpuacct_stat;
     struct cpuacct_usage cpuacct_usage;
+    struct cpuacct_cpu_throttling cpuacct_cpu_throttling;
 
     struct memory memory;
 
@@ -699,6 +756,9 @@ struct cgroup {
     RRDSET *st_cpu;
     RRDSET *st_cpu_limit;
     RRDSET *st_cpu_per_core;
+    RRDSET *st_cpu_nr_throttled;
+    RRDSET *st_cpu_throttled_time;
+
     RRDSET *st_mem;
     RRDSET *st_mem_utilization;
     RRDSET *st_writeback;
@@ -707,6 +767,7 @@ struct cgroup {
     RRDSET *st_mem_usage;
     RRDSET *st_mem_usage_limit;
     RRDSET *st_mem_failcnt;
+
     RRDSET *st_io;
     RRDSET *st_serviced_ops;
     RRDSET *st_throttle_io;
@@ -782,6 +843,22 @@ struct discovery_thread {
 } discovery_thread;
 
 // ----------------------------------------------------------------------------
+
+static unsigned long long calc_delta(unsigned long long curr, unsigned long long prev) {
+    if (prev > curr) {
+        return 0;
+    }
+    return curr - prev;
+}
+
+static unsigned long long calc_percentage(unsigned long long value, unsigned long long total) {
+    if (total == 0) {
+        return 0;
+    }
+    return (calculated_number)value / (calculated_number)total * 100;
+}
+
+// ----------------------------------------------------------------------------
 // read values from /sys
 
 static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
@@ -829,40 +906,126 @@ static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
     }
 }
 
-static inline void cgroup2_read_cpuacct_stat(struct cpuacct_stat *cp) {
+static inline void cgroup_read_cpuacct_cpu_stat(struct cpuacct_cpu_throttling *cp) {
+    if (unlikely(!cp->filename)) {
+        return;
+    }
+
     static procfile *ff = NULL;
+    ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
+    if (unlikely(!ff)) {
+        cp->updated = 0;
+        cgroups_check = 1;
+        return;
+    }
 
-    if(likely(cp->filename)) {
-        ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
-        if(unlikely(!ff)) {
-            cp->updated = 0;
-            cgroups_check = 1;
-            return;
-        }
+    ff = procfile_readall(ff);
+    if (unlikely(!ff)) {
+        cp->updated = 0;
+        cgroups_check = 1;
+        return;
+    }
 
-        ff = procfile_readall(ff);
-        if(unlikely(!ff)) {
-            cp->updated = 0;
-            cgroups_check = 1;
-            return;
+    unsigned long lines = procfile_lines(ff);
+    if (unlikely(lines < 3)) {
+        error("CGROUP: file '%s' should have 3 lines.", cp->filename);
+        cp->updated = 0;
+        return;
+    }
+
+    unsigned long long nr_periods_last = cp->nr_periods; 
+    unsigned long long nr_throttled_last = cp->nr_throttled; 
+
+    for (unsigned long i = 0; i < lines; i++) {
+        char *s = procfile_lineword(ff, i, 0);
+        uint32_t hash = simple_hash(s);
+
+        if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
+            cp->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
+            cp->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == throttled_time_hash && !strcmp(s, "throttled_time"))) {
+            cp->throttled_time = str2ull(procfile_lineword(ff, i, 1));
         }
+    }
+    cp->nr_throttled_perc =
+        calc_percentage(calc_delta(cp->nr_throttled, nr_throttled_last), calc_delta(cp->nr_periods, nr_periods_last));
 
-        unsigned long lines = procfile_lines(ff);
+    cp->updated = 1;
 
-        if(unlikely(lines < 3)) {
-            error("CGROUP: file '%s' should have 3+ lines.", cp->filename);
-            cp->updated = 0;
-            return;
+    if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
+        if (likely(
+                cp->nr_periods || cp->nr_throttled || cp->throttled_time ||
+                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+            cp->enabled = CONFIG_BOOLEAN_YES;
         }
+    }
+}
 
-        cp->user = str2ull(procfile_lineword(ff, 1, 1));
-        cp->system = str2ull(procfile_lineword(ff, 2, 1));
+static inline void cgroup2_read_cpuacct_cpu_stat(struct cpuacct_stat *cp, struct cpuacct_cpu_throttling *cpt) {
+    static procfile *ff = NULL;
+    if (unlikely(!cp->filename)) {
+        return;
+    }
 
-        cp->updated = 1;
+    ff = procfile_reopen(ff, cp->filename, NULL, PROCFILE_FLAG_DEFAULT);
+    if (unlikely(!ff)) {
+        cp->updated = 0;
+        cgroups_check = 1;
+        return;
+    }
 
-        if(unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO &&
-                    (cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)))
+    ff = procfile_readall(ff);
+    if (unlikely(!ff)) {
+        cp->updated = 0;
+        cgroups_check = 1;
+        return;
+    }
+
+    unsigned long lines = procfile_lines(ff);
+
+    if (unlikely(lines < 3)) {
+        error("CGROUP: file '%s' should have at least 3 lines.", cp->filename);
+        cp->updated = 0;
+        return;
+    }
+
+    unsigned long long nr_periods_last = cpt->nr_periods; 
+    unsigned long long nr_throttled_last = cpt->nr_throttled; 
+
+    for (unsigned long i = 0; i < lines; i++) {
+        char *s = procfile_lineword(ff, i, 0);
+        uint32_t hash = simple_hash(s);
+
+        if (unlikely(hash == user_usec_hash && !strcmp(s, "user_usec"))) {
+            cp->user = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == system_usec_hash && !strcmp(s, "system_usec"))) {
+            cp->system = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
+            cpt->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
+            cpt->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+        } else if (unlikely(hash == throttled_usec_hash && !strcmp(s, "throttled_usec"))) {
+            cpt->throttled_time = str2ull(procfile_lineword(ff, i, 1)) * 1000; // usec -> ns
+        }
+    }
+    cpt->nr_throttled_perc =
+        calc_percentage(calc_delta(cpt->nr_throttled, nr_throttled_last), calc_delta(cpt->nr_periods, nr_periods_last));
+
+    cp->updated = 1;
+    cpt->updated = 1;
+
+    if (unlikely(cp->enabled == CONFIG_BOOLEAN_AUTO)) {
+        if (likely(cp->user || cp->system || netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
             cp->enabled = CONFIG_BOOLEAN_YES;
+        }
+    }
+    if (unlikely(cpt->enabled == CONFIG_BOOLEAN_AUTO)) {
+        if (likely(
+                cpt->nr_periods || cpt->nr_throttled || cpt->throttled_time ||
+                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES)) {
+            cpt->enabled = CONFIG_BOOLEAN_YES;
+        }
     }
 }
 
@@ -1236,6 +1399,7 @@ static inline void cgroup_read(struct cgroup *cg) {
     if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
         cgroup_read_cpuacct_stat(&cg->cpuacct_stat);
         cgroup_read_cpuacct_usage(&cg->cpuacct_usage);
+        cgroup_read_cpuacct_cpu_stat(&cg->cpuacct_cpu_throttling);
         cgroup_read_memory(&cg->memory, 0);
         cgroup_read_blkio(&cg->io_service_bytes);
         cgroup_read_blkio(&cg->io_serviced);
@@ -1248,7 +1412,7 @@ static inline void cgroup_read(struct cgroup *cg) {
         //TODO: io_service_bytes and io_serviced use same file merge into 1 function
         cgroup2_read_blkio(&cg->io_service_bytes, 0);
         cgroup2_read_blkio(&cg->io_serviced, 4);
-        cgroup2_read_cpuacct_stat(&cg->cpuacct_stat);
+        cgroup2_read_cpuacct_cpu_stat(&cg->cpuacct_stat, &cg->cpuacct_cpu_throttling);
         cgroup2_read_pressure(&cg->cpu_pressure);
         cgroup2_read_pressure(&cg->io_pressure);
         cgroup2_read_pressure(&cg->memory_pressure);
@@ -1599,6 +1763,8 @@ static inline void cgroup_free(struct cgroup *cg) {
     if(cg->st_cpu)                   rrdset_is_obsolete(cg->st_cpu);
     if(cg->st_cpu_limit)             rrdset_is_obsolete(cg->st_cpu_limit);
     if(cg->st_cpu_per_core)          rrdset_is_obsolete(cg->st_cpu_per_core);
+    if(cg->st_cpu_nr_throttled)      rrdset_is_obsolete(cg->st_cpu_nr_throttled);
+    if(cg->st_cpu_throttled_time)    rrdset_is_obsolete(cg->st_cpu_throttled_time);
     if(cg->st_mem)                   rrdset_is_obsolete(cg->st_mem);
     if(cg->st_writeback)             rrdset_is_obsolete(cg->st_writeback);
     if(cg->st_mem_activity)          rrdset_is_obsolete(cg->st_mem_activity);
@@ -1626,6 +1792,7 @@ static inline void cgroup_free(struct cgroup *cg) {
 
     freez(cg->cpuacct_stat.filename);
     freez(cg->cpuacct_usage.filename);
+    freez(cg->cpuacct_cpu_throttling.filename);
 
     arl_free(cg->memory.arl_base);
     freez(cg->memory.filename_detailed);
@@ -1842,6 +2009,16 @@ static inline void update_filenames()
                 else
                     debug(D_CGROUP, "cpuacct.usage_percpu file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
+            if(unlikely(cgroup_enable_cpuacct_cpu_throttling && !cg->cpuacct_cpu_throttling.filename && !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE))) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_cpuacct_base, cg->id);
+                if(likely(stat(filename, &buf) != -1)) {
+                    cg->cpuacct_cpu_throttling.filename = strdupz(filename);
+                    cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
+                    debug(D_CGROUP, "cpu.stat filename for cgroup '%s': '%s'", cg->id, cg->cpuacct_cpu_throttling.filename);
+                }
+                else
+                    debug(D_CGROUP, "cpu.stat file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+            }
 
             if(unlikely((cgroup_enable_detailed_memory || cgroup_used_memory) && !cg->memory.filename_detailed && (cgroup_used_memory || cgroup_enable_systemd_services_detailed_memory || !(cg->options & CGROUP_OPTIONS_SYSTEM_SLICE_SERVICE)))) {
                 snprintfz(filename, FILENAME_MAX, "%s%s/memory.stat", cgroup_memory_base, cg->id);
@@ -1892,69 +2069,118 @@ static inline void update_filenames()
             }
 
             if(unlikely(cgroup_enable_blkio_io && !cg->io_service_bytes.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->io_service_bytes.filename = strdupz(filename);
                     cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
-                    debug(D_CGROUP, "io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+                    debug(D_CGROUP, "blkio.io_service_bytes_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.io_service_bytes_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_service_bytes", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->io_service_bytes.filename = strdupz(filename);
+                        cg->io_service_bytes.enabled = cgroup_enable_blkio_io;
+                        debug(D_CGROUP, "blkio.io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->io_service_bytes.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
 
-            if(unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+            if (unlikely(cgroup_enable_blkio_ops && !cg->io_serviced.filename)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->io_serviced.filename = strdupz(filename);
                     cg->io_serviced.enabled = cgroup_enable_blkio_ops;
-                    debug(D_CGROUP, "io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
+                    debug(D_CGROUP, "blkio.io_serviced_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.io_serviced_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_serviced", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->io_serviced.filename = strdupz(filename);
+                        cg->io_serviced.enabled = cgroup_enable_blkio_ops;
+                        debug(D_CGROUP, "blkio.io_serviced filename for cgroup '%s': '%s'", cg->id, cg->io_serviced.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
 
-            if(unlikely(cgroup_enable_blkio_throttle_io && !cg->throttle_io_service_bytes.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+            if (unlikely(cgroup_enable_blkio_throttle_io && !cg->throttle_io_service_bytes.filename)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->throttle_io_service_bytes.filename = strdupz(filename);
                     cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io;
-                    debug(D_CGROUP, "throttle_io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
+                    debug(D_CGROUP,"blkio.throttle.io_service_bytes_recursive filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.throttle.io_service_bytes_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(
+                        filename, FILENAME_MAX, "%s%s/blkio.throttle.io_service_bytes", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->throttle_io_service_bytes.filename = strdupz(filename);
+                        cg->throttle_io_service_bytes.enabled = cgroup_enable_blkio_throttle_io;
+                        debug(D_CGROUP, "blkio.throttle.io_service_bytes filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_service_bytes.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.throttle.io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "throttle_io_service_bytes file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
 
-            if(unlikely(cgroup_enable_blkio_throttle_ops && !cg->throttle_io_serviced.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+            if (unlikely(cgroup_enable_blkio_throttle_ops && !cg->throttle_io_serviced.filename)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->throttle_io_serviced.filename = strdupz(filename);
                     cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops;
-                    debug(D_CGROUP, "throttle_io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
+                    debug(D_CGROUP, "blkio.throttle.io_serviced_recursive filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.throttle.io_serviced_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(filename, FILENAME_MAX, "%s%s/blkio.throttle.io_serviced", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->throttle_io_serviced.filename = strdupz(filename);
+                        cg->throttle_io_serviced.enabled = cgroup_enable_blkio_throttle_ops;
+                        debug(D_CGROUP, "blkio.throttle.io_serviced filename for cgroup '%s': '%s'", cg->id, cg->throttle_io_serviced.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.throttle.io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "throttle_io_serviced file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
 
-            if(unlikely(cgroup_enable_blkio_merged_ops && !cg->io_merged.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+            if (unlikely(cgroup_enable_blkio_merged_ops && !cg->io_merged.filename)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->io_merged.filename = strdupz(filename);
                     cg->io_merged.enabled = cgroup_enable_blkio_merged_ops;
-                    debug(D_CGROUP, "io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
+                    debug(D_CGROUP, "blkio.io_merged_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.io_merged_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_merged", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->io_merged.filename = strdupz(filename);
+                        cg->io_merged.enabled = cgroup_enable_blkio_merged_ops;
+                        debug(D_CGROUP, "blkio.io_merged filename for cgroup '%s': '%s'", cg->id, cg->io_merged.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "io_merged file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
 
-            if(unlikely(cgroup_enable_blkio_queued_ops && !cg->io_queued.filename)) {
-                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id);
-                if(likely(stat(filename, &buf) != -1)) {
+            if (unlikely(cgroup_enable_blkio_queued_ops && !cg->io_queued.filename)) {
+                snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued_recursive", cgroup_blkio_base, cg->id);
+                if (unlikely(stat(filename, &buf) != -1)) {
                     cg->io_queued.filename = strdupz(filename);
                     cg->io_queued.enabled = cgroup_enable_blkio_queued_ops;
-                    debug(D_CGROUP, "io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
+                    debug(D_CGROUP, "blkio.io_queued_recursive filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
+                } else {
+                    debug(D_CGROUP, "blkio.io_queued_recursive file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    snprintfz(filename, FILENAME_MAX, "%s%s/blkio.io_queued", cgroup_blkio_base, cg->id);
+                    if (likely(stat(filename, &buf) != -1)) {
+                        cg->io_queued.filename = strdupz(filename);
+                        cg->io_queued.enabled = cgroup_enable_blkio_queued_ops;
+                        debug(D_CGROUP, "blkio.io_queued filename for cgroup '%s': '%s'", cg->id, cg->io_queued.filename);
+                    } else {
+                        debug(D_CGROUP, "blkio.io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename);
+                    }
                 }
-                else
-                    debug(D_CGROUP, "io_queued file for cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
         }
         else if(likely(cgroup_unified_exist)) {
@@ -1976,11 +2202,14 @@ static inline void update_filenames()
                 } else
                     debug(D_CGROUP, "io.stat file for unified cgroup '%s': '%s' does not exist.", cg->id, filename);
             }
-            if(unlikely(cgroup_enable_cpuacct_stat && !cg->cpuacct_stat.filename)) {
+            if (unlikely(
+                    (cgroup_enable_cpuacct_stat || cgroup_enable_cpuacct_cpu_throttling) &&
+                    !cg->cpuacct_stat.filename)) {
                 snprintfz(filename, FILENAME_MAX, "%s%s/cpu.stat", cgroup_unified_base, cg->id);
                 if(likely(stat(filename, &buf) != -1)) {
                     cg->cpuacct_stat.filename = strdupz(filename);
                     cg->cpuacct_stat.enabled = cgroup_enable_cpuacct_stat;
+                    cg->cpuacct_cpu_throttling.enabled = cgroup_enable_cpuacct_cpu_throttling;
                     cg->filename_cpuset_cpus = NULL;
                     cg->filename_cpu_cfs_period = NULL;
                     snprintfz(filename, FILENAME_MAX, "%s%s/cpu.max", cgroup_unified_base, cg->id);
@@ -3359,6 +3588,7 @@ void update_cgroup_charts(int update_every) {
                                 rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
                             else
                                 rrddim_add(cg->st_cpu_limit, "used", NULL, 1, 1000000, RRD_ALGORITHM_ABSOLUTE);
+                            cg->prev_cpu_usage = (calculated_number)(cg->cpuacct_stat.user + cg->cpuacct_stat.system) * 100;
                         }
                         else
                             rrdset_next(cg->st_cpu_limit);
@@ -3386,6 +3616,60 @@ void update_cgroup_charts(int update_every) {
             }
         }
 
+        if (likely(cg->cpuacct_cpu_throttling.updated && cg->cpuacct_cpu_throttling.enabled == CONFIG_BOOLEAN_YES)) {
+            if (unlikely(!cg->st_cpu_nr_throttled)) {
+                snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Runnable Periods");
+
+                cg->st_cpu_nr_throttled = rrdset_create_localhost(
+                        cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+                        , "throttled"
+                        , NULL
+                        , "cpu"
+                        , "cgroup.throttled"
+                        , title
+                        , "percentage"
+                        , PLUGIN_CGROUPS_NAME
+                        , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+                        , cgroup_containers_chart_priority + 10
+                        , update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels);
+                rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            } else {
+                rrdset_next(cg->st_cpu_nr_throttled);
+                rrddim_set(cg->st_cpu_nr_throttled, "throttled", cg->cpuacct_cpu_throttling.nr_throttled_perc);
+                rrdset_done(cg->st_cpu_nr_throttled);
+            }
+
+            if (unlikely(!cg->st_cpu_throttled_time)) {
+                snprintfz(title, CHART_TITLE_MAX, "CPU Throttled Time Duration");
+
+                cg->st_cpu_throttled_time = rrdset_create_localhost(
+                        cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX)
+                        , "throttled_duration"
+                        , NULL
+                        , "cpu"
+                        , "cgroup.throttled_duration"
+                        , title
+                        , "ms"
+                        , PLUGIN_CGROUPS_NAME
+                        , PLUGIN_CGROUPS_MODULE_CGROUPS_NAME
+                        , cgroup_containers_chart_priority + 15
+                        , update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels);
+                rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
+            } else {
+                rrdset_next(cg->st_cpu_throttled_time);
+                rrddim_set(cg->st_cpu_throttled_time, "duration", cg->cpuacct_cpu_throttling.throttled_time);
+                rrdset_done(cg->st_cpu_throttled_time);
+            }
+        }
+
         if(likely(cg->cpuacct_usage.updated && cg->cpuacct_usage.enabled == CONFIG_BOOLEAN_YES)) {
             char id[RRD_ID_LENGTH_MAX + 1];
             unsigned int i;