Merging upstream version 1.41.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 493 insertions, 5 deletions

diff --git a/collectors/ebpf.plugin/ebpf_vfs.c b/collectors/ebpf.plugin/ebpf_vfs.c
index 6cafafc38..5747a2408 100644
--- a/collectors/ebpf.plugin/ebpf_vfs.c
+++ b/collectors/ebpf.plugin/ebpf_vfs.c
@@ -60,6 +60,10 @@ netdata_ebpf_targets_t vfs_targets[] = { {.name = "vfs_write", .mode = EBPF_LOAD
                                          {.name = "release_task", .mode = EBPF_LOAD_TRAMPOLINE},
                                          {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
 
+#ifdef NETDATA_DEV_MODE
+int vfs_disable_priority;
+#endif
+
 #ifdef LIBBPF_MAJOR_VERSION
 /**
  * Disable probe
@@ -403,6 +407,447 @@ static inline int ebpf_vfs_load_and_attach(struct vfs_bpf *obj, ebpf_module_t *e
  *
  *****************************************************************/
 
+static void ebpf_obsolete_specific_vfs_charts(char *type, ebpf_module_t *em);
+
+/**
+ * Obsolete services
+ *
+ * Obsolete all service charts created
+ *
+ * @param em a pointer to `struct ebpf_module`
+ */
+static void ebpf_obsolete_vfs_services(ebpf_module_t *em)
+{
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_FILE_DELETED,
+                              "Files deleted",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20065,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS,
+                              "Write to disk",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20066,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR,
+                                  "Fails to write",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_CGROUP_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20067,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_READ_CALLS,
+                              "Read from disk",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20068,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR,
+                                  "Fails to read",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_CGROUP_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20069,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES,
+                              "Bytes written on disk",
+                              EBPF_COMMON_DIMENSION_BYTES,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20070,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_READ_BYTES,
+                              "Bytes read from disk",
+                              EBPF_COMMON_DIMENSION_BYTES,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20071,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_FSYNC,
+                              "Calls to <code>vfs_fsync</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20072,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR,
+                                  "Sync error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_CGROUP_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20073,
+                                  em->update_every);
+    }
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_OPEN,
+                              "Calls to <code>vfs_open</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20074,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR,
+                                  "Open error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_CGROUP_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20075,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_CREATE,
+                              "Calls to <code>vfs_create</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_CGROUP_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20076,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_SERVICE_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR,
+                                  "Create error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_CGROUP_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20077,
+                                  em->update_every);
+    }
+}
+
+/**
+ * Obsolete cgroup chart
+ *
+ * Send obsolete for all charts created before to close.
+ *
+ * @param em a pointer to `struct ebpf_module`
+ */
+static inline void ebpf_obsolete_vfs_cgroup_charts(ebpf_module_t *em) {
+    pthread_mutex_lock(&mutex_cgroup_shm);
+
+    ebpf_obsolete_vfs_services(em);
+
+    ebpf_cgroup_target_t *ect;
+    for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
+        if (ect->systemd)
+            continue;
+
+        ebpf_obsolete_specific_vfs_charts(ect->name, em);
+    }
+    pthread_mutex_unlock(&mutex_cgroup_shm);
+}
+
+/**
+ * Obsolette apps charts
+ *
+ * Obsolete apps charts.
+ *
+ * @param em a pointer to the structure with the default values.
+ */
+void ebpf_obsolete_vfs_apps_charts(struct ebpf_module *em)
+{
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_FILE_DELETED,
+                              "Files deleted",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20065,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS,
+                              "Write to disk",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20066,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_WRITE_CALLS_ERROR,
+                                  "Fails to write",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20067,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_READ_CALLS,
+                              "Read from disk",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20068,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_READ_CALLS_ERROR,
+                                  "Fails to read",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20069,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_WRITE_BYTES,
+                              "Bytes written on disk",
+                              EBPF_COMMON_DIMENSION_BYTES,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20070,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_READ_BYTES,
+                              "Bytes read from disk",
+                              EBPF_COMMON_DIMENSION_BYTES,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20071,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_FSYNC,
+                              "Calls for <code>vfs_fsync</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20072,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_FSYNC_CALLS_ERROR,
+                                  "Sync error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20073,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_OPEN,
+                              "Calls for <code>vfs_open</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20074,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_OPEN_CALLS_ERROR,
+                                  "Open error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20075,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                              NETDATA_SYSCALL_APPS_VFS_CREATE,
+                              "Calls for <code>vfs_create</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_STACKED,
+                              NULL,
+                              20076,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_APPS_FAMILY,
+                                  NETDATA_SYSCALL_APPS_VFS_CREATE_CALLS_ERROR,
+                                  "Create error",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_STACKED,
+                                  NULL,
+                                  20077,
+                                  em->update_every);
+    }
+}
+
+/**
+ * Obsolete global
+ *
+ * Obsolete global charts created by thread.
+ *
+ * @param em a pointer to `struct ebpf_module`
+ */
+static void ebpf_obsolete_vfs_global(ebpf_module_t *em)
+{
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_FILE_CLEAN_COUNT,
+                              "Remove files",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_CLEAN,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_FILE_IO_COUNT,
+                              "Calls to IO",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_COUNT,
+                              em->update_every);
+
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_IO_FILE_BYTES,
+                              "Bytes written and read",
+                              EBPF_COMMON_DIMENSION_BYTES,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_BYTES,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                                  NETDATA_VFS_FILE_ERR_COUNT,
+                                  "Fails to write or read",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_LINE,
+                                  NULL,
+                                  NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EBYTES,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_FSYNC,
+                              "Calls for <code>vfs_fsync</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_FSYNC,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                                  NETDATA_VFS_FSYNC_ERR,
+                                  "Fails to synchronize",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_LINE,
+                                  NULL,
+                                  NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EFSYNC,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_OPEN,
+                              "Calls for <code>vfs_open</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_OPEN,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                                  NETDATA_VFS_OPEN_ERR,
+                                  "Fails to open a file",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_LINE,
+                                  NULL,
+                                  NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_EOPEN,
+                                  em->update_every);
+    }
+
+    ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                              NETDATA_VFS_CREATE,
+                              "Calls for <code>vfs_create</code>",
+                              EBPF_COMMON_DIMENSION_CALL,
+                              NETDATA_VFS_GROUP,
+                              NETDATA_EBPF_CHART_TYPE_LINE,
+                              NULL,
+                              NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_CREATE,
+                              em->update_every);
+
+    if (em->mode < MODE_ENTRY) {
+        ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY,
+                                  NETDATA_VFS_CREATE_ERR,
+                                  "Fails to create a file.",
+                                  EBPF_COMMON_DIMENSION_CALL,
+                                  NETDATA_VFS_GROUP,
+                                  NETDATA_EBPF_CHART_TYPE_LINE,
+                                  NULL,
+                                  NETDATA_CHART_PRIO_FILESYSTEM_VFS_IO_ECREATE,
+                                  em->update_every);
+    }
+}
+
 /**
  * Exit
  *
@@ -414,15 +859,45 @@ static void ebpf_vfs_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
+    if (em->enabled == NETDATA_THREAD_EBPF_FUNCTION_RUNNING) {
+        pthread_mutex_lock(&lock);
+        if (em->cgroup_charts) {
+            ebpf_obsolete_vfs_cgroup_charts(em);
+            fflush(stdout);
+        }
+
+        if (em->apps_charts & NETDATA_EBPF_APPS_FLAG_CHART_CREATED) {
+            ebpf_obsolete_vfs_apps_charts(em);
+        }
+
+        ebpf_obsolete_vfs_global(em);
+
+#ifdef NETDATA_DEV_MODE
+        if (ebpf_aral_vfs_pid)
+            ebpf_statistic_obsolete_aral_chart(em, vfs_disable_priority);
+#endif
+
+        fflush(stdout);
+        pthread_mutex_unlock(&lock);
+    }
+
+    ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps, EBPF_ACTION_STAT_REMOVE);
+
 #ifdef LIBBPF_MAJOR_VERSION
-    if (vfs_bpf_obj)
+    if (vfs_bpf_obj) {
         vfs_bpf__destroy(vfs_bpf_obj);
+        vfs_bpf_obj = NULL;
+    }
 #endif
-    if (em->objects)
+    if (em->objects) {
         ebpf_unload_legacy_code(em->objects, em->probe_links);
+        em->objects = NULL;
+        em->probe_links = NULL;
+    }
 
     pthread_mutex_lock(&ebpf_exit_cleanup);
     em->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    ebpf_update_stats(&plugin_statistics, em);
     pthread_mutex_unlock(&ebpf_exit_cleanup);
 }
 
@@ -1486,7 +1961,9 @@ static void vfs_collector(ebpf_module_t *em)
     int update_every = em->update_every;
     int counter = update_every - 1;
     int maps_per_core = em->maps_per_core;
-    while (!ebpf_exit_plugin) {
+    uint32_t running_time = 0;
+    uint32_t lifetime = em->lifetime;
+    while (!ebpf_exit_plugin && running_time < lifetime) {
         (void)heartbeat_next(&hb, USEC_PER_SEC);
         if (ebpf_exit_plugin || ++counter != update_every)
             continue;
@@ -1519,6 +1996,15 @@ static void vfs_collector(ebpf_module_t *em)
 
         pthread_mutex_unlock(&lock);
         pthread_mutex_unlock(&collect_data_mutex);
+
+        pthread_mutex_lock(&ebpf_exit_cleanup);
+        if (running_time && !em->running_time)
+            running_time = update_every;
+        else
+            running_time += update_every;
+
+        em->running_time = running_time;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
     }
 }
 
@@ -1690,6 +2176,8 @@ static void ebpf_create_global_charts(ebpf_module_t *em)
                           &vfs_publish_aggregated[NETDATA_KEY_PUBLISH_VFS_CREATE],
                           1, em->update_every, NETDATA_EBPF_MODULE_NAME_VFS);
     }
+
+    fflush(stdout);
 }
 
 /**
@@ -1934,10 +2422,10 @@ void *ebpf_vfs_thread(void *ptr)
     pthread_mutex_lock(&lock);
     ebpf_create_global_charts(em);
     ebpf_update_stats(&plugin_statistics, em);
-    ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps);
+    ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps, EBPF_ACTION_STAT_ADD);
 #ifdef NETDATA_DEV_MODE
     if (ebpf_aral_vfs_pid)
-        ebpf_statistic_create_aral_chart(NETDATA_EBPF_VFS_ARAL_NAME, em);
+        vfs_disable_priority = ebpf_statistic_create_aral_chart(NETDATA_EBPF_VFS_ARAL_NAME, em);
 #endif
 
     pthread_mutex_unlock(&lock);