Merging upstream version 2.0.3+dfsg (Closes: #923993, #1042533, #1045145).

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 327 insertions, 137 deletions

diff --git a/src/collectors/apps.plugin/apps_targets.c b/src/collectors/apps.plugin/apps_targets.c
index 7deaa798c..46db128cc 100644
--- a/src/collectors/apps.plugin/apps_targets.c
+++ b/src/collectors/apps.plugin/apps_targets.c
@@ -2,199 +2,370 @@
 
 #include "apps_plugin.h"
 
-// ----------------------------------------------------------------------------
-// apps_groups.conf
-// aggregate all processes in groups, to have a limited number of dimensions
+pid_t INIT_PID = OS_INIT_PID;
 
-struct target *get_users_target(uid_t uid) {
-    struct target *w;
-    for(w = users_root_target ; w ; w = w->next)
-        if(w->uid == uid) return w;
+static STRING *get_clean_name(STRING *name) {
+    char buf[string_strlen(name) + 1];
+    memcpy(buf, string2str(name), string_strlen(name) + 1);
+    netdata_fix_chart_name(buf);
 
-    w = callocz(sizeof(struct target), 1);
-    snprintfz(w->compare, MAX_COMPARE_NAME, "%u", uid);
-    w->comparehash = simple_hash(w->compare);
-    w->comparelen = strlen(w->compare);
+    for (char *d = buf; *d ; d++)
+        if (*d == '.') *d = '_';
 
-    snprintfz(w->id, MAX_NAME, "%u", uid);
-    w->idhash = simple_hash(w->id);
+    return string_strdupz(buf);
+}
 
-    struct user_or_group_id user_id_to_find = {
-        .id = {
-            .uid = uid,
-        }
-    };
-    struct user_or_group_id *user_or_group_id = user_id_find(&user_id_to_find);
+static inline STRING *get_numeric_string(uint64_t n) {
+    char buf[UINT64_MAX_LENGTH];
+    print_uint64(buf, n);
+    return string_strdupz(buf);
+}
 
-    if(user_or_group_id && user_or_group_id->name && *user_or_group_id->name)
-        snprintfz(w->name, MAX_NAME, "%s", user_or_group_id->name);
+struct target *find_target_by_name(struct target *base, const char *name) {
+    struct target *t;
+    for(t = base; t ; t = t->next) {
+        if (string_strcmp(t->name, name) == 0)
+            return t;
+    }
 
-    else {
-        struct passwd *pw = getpwuid(uid);
-        if(!pw || !pw->pw_name || !*pw->pw_name)
-            snprintfz(w->name, MAX_NAME, "%u", uid);
+    return NULL;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Process managers and aggregators
+
+struct comm_list {
+    APPS_MATCH match;
+};
+
+struct managed_list {
+    size_t used;
+    size_t size;
+    struct comm_list *array;
+};
+
+static struct {
+    struct managed_list managers;
+    struct managed_list aggregators;
+    struct managed_list interpreters;
+} tree = {
+    .managers = {
+        .array = NULL,
+        .size = 0,
+        .used = 0,
+    },
+    .aggregators = {
+        .array = NULL,
+        .size = 0,
+        .used = 0,
+    }
+};
+
+static void managed_list_clear(struct managed_list *list) {
+    for(size_t c = 0; c < list->used ; c++)
+        pid_match_cleanup(&list->array[c].match);
+
+    freez(list->array);
+    list->array = NULL;
+    list->used = 0;
+    list->size = 0;
+}
+
+static void managed_list_add(struct managed_list *list, const char *s) {
+    if(list->used >= list->size) {
+        if(!list->size)
+            list->size = 16;
         else
-            snprintfz(w->name, MAX_NAME, "%s", pw->pw_name);
+            list->size *= 2;
+        list->array = reallocz(list->array, sizeof(*list->array) * list->size);
+    }
+
+    list->array[list->used++].match = pid_match_create(s);
+}
+
+static STRING *KernelAggregator = NULL;
+
+void apps_managers_and_aggregators_init(void) {
+    KernelAggregator = string_strdupz("kernel");
+
+    managed_list_clear(&tree.managers);
+#if defined(OS_LINUX)
+    managed_list_add(&tree.managers, "init");                       // linux systems
+    managed_list_add(&tree.managers, "systemd");                    // lxc containers and host systems (this also catches "systemd --user")
+    managed_list_add(&tree.managers, "containerd-shim-runc-v2");    // docker containers
+    managed_list_add(&tree.managers, "docker-init");                // docker containers
+    managed_list_add(&tree.managers, "tini");                       // docker containers (https://github.com/krallin/tini)
+    managed_list_add(&tree.managers, "dumb-init");                  // some docker containers use this
+    managed_list_add(&tree.managers, "openrc-run.sh");              // openrc
+    managed_list_add(&tree.managers, "crond");                      // linux crond
+    managed_list_add(&tree.managers, "gnome-shell");                // gnome user applications
+    managed_list_add(&tree.managers, "plasmashell");                // kde user applications
+    managed_list_add(&tree.managers, "xfwm4");                      // xfce4 user applications
+#elif defined(OS_WINDOWS)
+    managed_list_add(&tree.managers, "wininit");
+    managed_list_add(&tree.managers, "services");
+    managed_list_add(&tree.managers, "explorer");
+    managed_list_add(&tree.managers, "System");
+#elif defined(OS_FREEBSD)
+    managed_list_add(&tree.managers, "init");
+#elif defined(OS_MACOS)
+    managed_list_add(&tree.managers, "launchd");
+#endif
+
+#if defined(OS_WINDOWS)
+    managed_list_add(&tree.managers, "netdata");
+#else
+    managed_list_add(&tree.managers, "spawn-plugins");
+#endif
+
+    managed_list_clear(&tree.aggregators);
+#if defined(OS_LINUX)
+    managed_list_add(&tree.aggregators, "kthread");
+#elif defined(OS_WINDOWS)
+#elif defined(OS_FREEBSD)
+    managed_list_add(&tree.aggregators, "kernel");
+#elif defined(OS_MACOS)
+#endif
+
+    managed_list_clear(&tree.interpreters);
+    managed_list_add(&tree.interpreters, "python");
+    managed_list_add(&tree.interpreters, "python2");
+    managed_list_add(&tree.interpreters, "python3");
+    managed_list_add(&tree.interpreters, "sh");
+    managed_list_add(&tree.interpreters, "bash");
+    managed_list_add(&tree.interpreters, "node");
+    managed_list_add(&tree.interpreters, "perl");
+}
+
+bool is_process_a_manager(struct pid_stat *p) {
+    for(size_t c = 0; c < tree.managers.used ; c++) {
+        if(pid_match_check(p, &tree.managers.array[c].match))
+            return true;
+    }
+
+    return false;
+}
+
+bool is_process_an_aggregator(struct pid_stat *p) {
+    for(size_t c = 0; c < tree.aggregators.used ; c++) {
+        if(pid_match_check(p, &tree.aggregators.array[c].match))
+            return true;
+    }
+
+    return false;
+}
+
+bool is_process_an_interpreter(struct pid_stat *p) {
+    for(size_t c = 0; c < tree.interpreters.used ; c++) {
+        if(pid_match_check(p, &tree.interpreters.array[c].match))
+            return true;
+    }
+
+    return false;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Tree
+
+struct target *get_tree_target(struct pid_stat *p) {
+//    // skip fast all the children that are more than 3 levels down
+//    while(p->parent && p->parent->pid != INIT_PID && p->parent->parent && p->parent->parent->parent)
+//        p = p->parent;
+
+    // keep the children of INIT_PID, and process orchestrators
+    while(p->parent && p->parent->pid != INIT_PID && p->parent->pid != 0 && !p->parent->is_manager)
+        p = p->parent;
+
+    // merge all processes into process aggregators
+    STRING *search_for = NULL;
+    if((p->ppid == 0 && p->pid != INIT_PID) || (p->parent && p->parent->is_aggregator)) {
+        search_for = string_dup(KernelAggregator);
     }
+    else {
+#if (PROCESSES_HAVE_COMM_AND_NAME == 1)
+        search_for = string_dup(p->name ? p->name : p->comm);
+#else
+        search_for = string_dup(p->comm);
+#endif
+    }
+
+    // find an existing target with the required name
+    struct target *w;
+    for(w = apps_groups_root_target; w ; w = w->next) {
+        if (w->name == search_for) {
+            string_freez(search_for);
+            return w;
+        }
+    }
+
+    w = callocz(sizeof(struct target), 1);
+    w->type = TARGET_TYPE_TREE;
+    w->match.starts_with = w->match.ends_with = false;
+    w->match.compare = string_dup(search_for);
+    w->match.pattern = NULL;
+    w->id = search_for;
+    w->name = string_dup(search_for);
+    w->clean_name = get_clean_name(w->name);
+
+    w->next = apps_groups_root_target;
+    apps_groups_root_target = w;
+
+    return w;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+// Users
+
+#if (PROCESSES_HAVE_UID == 1)
+struct target *users_root_target = NULL;
 
-    strncpyz(w->clean_name, w->name, MAX_NAME);
-    netdata_fix_chart_name(w->clean_name);
+struct target *get_uid_target(uid_t uid) {
+    struct target *w;
+    for(w = users_root_target ; w ; w = w->next)
+        if(w->uid == uid) return w;
 
+    w = callocz(sizeof(struct target), 1);
+    w->type = TARGET_TYPE_UID;
     w->uid = uid;
+    w->id = get_numeric_string(uid);
+
+    CACHED_USERNAME cu = cached_username_get_by_uid(uid);
+    w->name = string_dup(cu.username);
+    w->clean_name = get_clean_name(w->name);
+    cached_username_release(cu);
 
     w->next = users_root_target;
     users_root_target = w;
 
-    debug_log("added uid %u ('%s') target", w->uid, w->name);
+    debug_log("added uid %u ('%s') target", w->uid, string2str(w->name));
 
     return w;
 }
+#endif
+
+// --------------------------------------------------------------------------------------------------------------------
+// Groups
+
+#if (PROCESSES_HAVE_GID == 1)
+struct target *groups_root_target = NULL;
 
-struct target *get_groups_target(gid_t gid) {
+struct target *get_gid_target(gid_t gid) {
     struct target *w;
     for(w = groups_root_target ; w ; w = w->next)
         if(w->gid == gid) return w;
 
     w = callocz(sizeof(struct target), 1);
-    snprintfz(w->compare, MAX_COMPARE_NAME, "%u", gid);
-    w->comparehash = simple_hash(w->compare);
-    w->comparelen = strlen(w->compare);
+    w->type = TARGET_TYPE_GID;
+    w->gid = gid;
+    w->id = get_numeric_string(gid);
 
-    snprintfz(w->id, MAX_NAME, "%u", gid);
-    w->idhash = simple_hash(w->id);
+    CACHED_GROUPNAME cg = cached_groupname_get_by_gid(gid);
+    w->name = string_dup(cg.groupname);
+    w->clean_name = get_clean_name(w->name);
+    cached_groupname_release(cg);
 
-    struct user_or_group_id group_id_to_find = {
-        .id = {
-            .gid = gid,
-        }
-    };
-    struct user_or_group_id *group_id = group_id_find(&group_id_to_find);
+    w->next = groups_root_target;
+    groups_root_target = w;
 
-    if(group_id && group_id->name && *group_id->name) {
-        snprintfz(w->name, MAX_NAME, "%s", group_id->name);
-    }
-    else {
-        struct group *gr = getgrgid(gid);
-        if(!gr || !gr->gr_name || !*gr->gr_name)
-            snprintfz(w->name, MAX_NAME, "%u", gid);
-        else
-            snprintfz(w->name, MAX_NAME, "%s", gr->gr_name);
-    }
+    debug_log("added gid %u ('%s') target", w->gid, w->name);
 
-    strncpyz(w->clean_name, w->name, MAX_NAME);
-    netdata_fix_chart_name(w->clean_name);
+    return w;
+}
+#endif
 
-    w->gid = gid;
+// --------------------------------------------------------------------------------------------------------------------
+// SID
 
-    w->next = groups_root_target;
-    groups_root_target = w;
+#if (PROCESSES_HAVE_SID == 1)
+struct target *sids_root_target = NULL;
 
-    debug_log("added gid %u ('%s') target", w->gid, w->name);
+struct target *get_sid_target(STRING *sid_name) {
+    struct target *w;
+    for(w = sids_root_target ; w ; w = w->next)
+        if(w->sid_name == sid_name) return w;
+
+    w = callocz(sizeof(struct target), 1);
+    w->type = TARGET_TYPE_SID;
+    w->sid_name = string_dup(sid_name);
+    w->id = string_dup(sid_name);
+    w->name = string_dup(sid_name);
+    w->clean_name = get_clean_name(w->name);
+
+    w->next = sids_root_target;
+    sids_root_target = w;
+
+    debug_log("added uid %s ('%s') target", string2str(w->sid_name), string2str(w->name));
 
     return w;
 }
+#endif
+
+// --------------------------------------------------------------------------------------------------------------------
+// apps_groups.conf
+
+struct target *apps_groups_root_target = NULL;
 
 // find or create a new target
 // there are targets that are just aggregated to other target (the second argument)
-static struct target *get_apps_groups_target(const char *id, struct target *target, const char *name) {
-    int tdebug = 0, thidden = target?target->hidden:0, ends_with = 0;
-    const char *nid = id;
-
-    // extract the options
-    while(nid[0] == '-' || nid[0] == '+' || nid[0] == '*') {
-        if(nid[0] == '-') thidden = 1;
-        if(nid[0] == '+') tdebug = 1;
-        if(nid[0] == '*') ends_with = 1;
-        nid++;
-    }
-    uint32_t hash = simple_hash(id);
+static struct target *get_apps_groups_target(const char *comm, struct target *target, const char *name) {
+    APPS_MATCH match = pid_match_create(comm);
+    STRING *name_lookup = string_strdupz(name);
 
     // find if it already exists
     struct target *w, *last = apps_groups_root_target;
     for(w = apps_groups_root_target ; w ; w = w->next) {
-        if(w->idhash == hash && strncmp(nid, w->id, MAX_NAME) == 0)
+        if(w->id == match.compare) {
+            pid_match_cleanup(&match);
+            string_freez(name_lookup);
             return w;
+        }
 
         last = w;
     }
 
     // find an existing target
     if(unlikely(!target)) {
-        while(*name == '-') {
-            if(*name == '-') thidden = 1;
-            name++;
-        }
-
-        for(target = apps_groups_root_target ; target != NULL ; target = target->next) {
-            if(!target->target && strcmp(name, target->name) == 0)
+        for(target = apps_groups_root_target ; target ; target = target->next) {
+            if(!target->target && name_lookup == target->name)
                 break;
         }
-
-        if(unlikely(debug_enabled)) {
-            if(unlikely(target))
-                debug_log("REUSING TARGET NAME '%s' on ID '%s'", target->name, target->id);
-            else
-                debug_log("NEW TARGET NAME '%s' on ID '%s'", name, id);
-        }
     }
 
     if(target && target->target)
-        fatal("Internal Error: request to link process '%s' to target '%s' which is linked to target '%s'", id, target->id, target->target->id);
+        fatal("Internal Error: request to link process '%s' to target '%s' which is linked to target '%s'",
+            comm, string2str(target->id), string2str(target->target->id));
 
     w = callocz(sizeof(struct target), 1);
-    strncpyz(w->id, nid, MAX_NAME);
-    w->idhash = simple_hash(w->id);
+    w->type = TARGET_TYPE_APP_GROUP;
+    w->match = match;
+    w->id = string_dup(w->match.compare);
 
     if(unlikely(!target))
-        // copy the name
-        strncpyz(w->name, name, MAX_NAME);
+        w->name = string_dup(name_lookup); // copy the name
     else
-        // copy the id
-        strncpyz(w->name, nid, MAX_NAME);
+        w->name = string_dup(w->id); // copy the id
 
     // dots are used to distinguish chart type and id in streaming, so we should replace them
-    strncpyz(w->clean_name, w->name, MAX_NAME);
-    netdata_fix_chart_name(w->clean_name);
-    for (char *d = w->clean_name; *d; d++) {
-        if (*d == '.')
-            *d = '_';
-    }
-
-    strncpyz(w->compare, nid, MAX_COMPARE_NAME);
-    size_t len = strlen(w->compare);
-    if(w->compare[len - 1] == '*') {
-        w->compare[len - 1] = '\0';
-        w->starts_with = 1;
-    }
-    w->ends_with = ends_with;
+    w->clean_name = get_clean_name(w->name);
 
-    if(w->starts_with && w->ends_with)
+    if(w->match.starts_with && w->match.ends_with)
         proc_pid_cmdline_is_needed = true;
 
-    w->comparehash = simple_hash(w->compare);
-    w->comparelen = strlen(w->compare);
-
-    w->hidden = thidden;
-#ifdef NETDATA_INTERNAL_CHECKS
-    w->debug_enabled = tdebug;
-#else
-    if(tdebug)
-        fprintf(stderr, "apps.plugin has been compiled without debugging\n");
-#endif
     w->target = target;
 
     // append it, to maintain the order in apps_groups.conf
     if(last) last->next = w;
     else apps_groups_root_target = w;
 
-    debug_log("ADDING TARGET ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s"
-              , w->id
-              , w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact"))
-                  , w->target?w->target->name:w->name
-              , (w->hidden)?"hidden":"-"
-              , (w->debug_enabled)?"debug":"-"
+    debug_log("ADDING TARGET ID '%s', process name '%s' (%s), aggregated on target '%s'"
+              , string2str(w->id)
+              , string2str(w->match.compare)
+              , (w->match.starts_with && w->match.ends_with) ? "substring" : ((w->match.starts_with) ? "prefix" : ((w->match.ends_with) ? "suffix" : "exact"))
+              , w->target?w->target->name:w->name
     );
 
+    string_freez(name_lookup);
+
     return w;
 }
 
@@ -224,20 +395,49 @@ int read_apps_groups_conf(const char *path, const char *file) {
         if(!words) continue;
 
         char *name = procfile_lineword(ff, line, 0);
-        if(!name || !*name) continue;
+        if(!name || !*name || *name == '#') continue;
+
+        if(strcmp(name, "managers") == 0) {
+            if(words == 2 && strcmp(procfile_lineword(ff, line, 1), "clear") == 0)
+                managed_list_clear(&tree.managers);
+
+            for(word = 1; word < words ;word++) {
+                char *s = procfile_lineword(ff, line, word);
+                if (!s || !*s) continue;
+                if (*s == '#') break;
+
+                managed_list_add(&tree.managers, s);
+            }
+
+            // done with managers, proceed to next line
+            continue;
+        }
+
+        if(strcmp(name, "interpreters") == 0) {
+            if(words == 2 && strcmp(procfile_lineword(ff, line, 1), "clear") == 0)
+                managed_list_clear(&tree.interpreters);
+
+            for(word = 1; word < words ;word++) {
+                char *s = procfile_lineword(ff, line, word);
+                if (!s || !*s) continue;
+                if (*s == '#') break;
+
+                managed_list_add(&tree.interpreters, s);
+            }
+
+            // done with managers, proceed to the next line
+            continue;
+        }
 
         // find a possibly existing target
         struct target *w = NULL;
 
         // loop through all words, skipping the first one (the name)
-        for(word = 0; word < words ;word++) {
+        for(word = 1; word < words ;word++) {
             char *s = procfile_lineword(ff, line, word);
             if(!s || !*s) continue;
             if(*s == '#') break;
 
-            // is this the first word? skip it
-            if(s == name) continue;
-
             // add this target
             struct target *n = get_apps_groups_target(s, w, name);
             if(!n) {
@@ -252,15 +452,5 @@ int read_apps_groups_conf(const char *path, const char *file) {
     }
 
     procfile_close(ff);
-
-    apps_groups_default_target = get_apps_groups_target("p+!o@w#e$i^r&7*5(-i)l-o_", NULL, "other"); // match nothing
-    if(!apps_groups_default_target)
-        fatal("Cannot create default target");
-    apps_groups_default_target->is_other = true;
-
-    // allow the user to override group 'other'
-    if(apps_groups_default_target->target)
-        apps_groups_default_target = apps_groups_default_target->target;
-
     return 0;
 }