Adding upstream version 3.1.8.upstream/3.1.8upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 784 insertions, 0 deletions

diff --git a/exec/stats.c b/exec/stats.c
new file mode 100644
index 0000000..d9fd115
--- /dev/null
+++ b/exec/stats.c
@@ -0,0 +1,784 @@
+/*
+ * Copyright (c) 2017-2020 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Authors: Christine Caulfield (ccaulfie@redhat.com)
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <libknet.h>
+
+#include <qb/qblist.h>
+#include <qb/qbipcs.h>
+#include <qb/qbipc_common.h>
+
+#include <corosync/corodefs.h>
+#include <corosync/coroapi.h>
+#include <corosync/logsys.h>
+#include <corosync/icmap.h>
+#include <corosync/totem/totemstats.h>
+
+#include "util.h"
+#include "ipcs_stats.h"
+#include "stats.h"
+
+LOGSYS_DECLARE_SUBSYS ("STATS");
+
+static qb_map_t *stats_map;
+
+/* Structure of an element in the schedmiss array */
+struct schedmiss_entry {
+	uint64_t timestamp;
+	float delay;
+};
+#define MAX_SCHEDMISS_EVENTS 10
+static struct schedmiss_entry schedmiss_event[MAX_SCHEDMISS_EVENTS];
+static unsigned int highest_schedmiss_event;
+
+#define SCHEDMISS_PREFIX "stats.schedmiss"
+
+/* Convert iterator number to text and a stats pointer */
+struct cs_stats_conv {
+	enum {STAT_PG, STAT_SRP, STAT_KNET, STAT_KNET_HANDLE, STAT_IPCSC, STAT_IPCSG, STAT_SCHEDMISS} type;
+	const char *name;
+	const size_t offset;
+	const icmap_value_types_t value_type;
+};
+
+struct cs_stats_conv cs_pg_stats[] = {
+	{ STAT_PG, "msg_queue_avail",         offsetof(totempg_stats_t, msg_queue_avail),         ICMAP_VALUETYPE_UINT32},
+	{ STAT_PG, "msg_reserved",            offsetof(totempg_stats_t, msg_reserved),            ICMAP_VALUETYPE_UINT32},
+};
+struct cs_stats_conv cs_srp_stats[] = {
+	{ STAT_SRP, "orf_token_tx",           offsetof(totemsrp_stats_t, orf_token_tx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "orf_token_rx",           offsetof(totemsrp_stats_t, orf_token_rx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_merge_detect_tx",   offsetof(totemsrp_stats_t, memb_merge_detect_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_merge_detect_rx",   offsetof(totemsrp_stats_t, memb_merge_detect_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_join_tx",           offsetof(totemsrp_stats_t, memb_join_tx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_join_rx",           offsetof(totemsrp_stats_t, memb_join_rx),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_tx",               offsetof(totemsrp_stats_t, mcast_tx),               ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_retx",             offsetof(totemsrp_stats_t, mcast_retx),             ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "mcast_rx",               offsetof(totemsrp_stats_t, mcast_rx),               ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_commit_token_tx",   offsetof(totemsrp_stats_t, memb_commit_token_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "memb_commit_token_rx",   offsetof(totemsrp_stats_t, memb_commit_token_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "token_hold_cancel_tx",   offsetof(totemsrp_stats_t, token_hold_cancel_tx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "token_hold_cancel_rx",   offsetof(totemsrp_stats_t, token_hold_cancel_rx),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "operational_entered",    offsetof(totemsrp_stats_t, operational_entered),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "operational_token_lost", offsetof(totemsrp_stats_t, operational_token_lost), ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "gather_entered",         offsetof(totemsrp_stats_t, gather_entered),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "gather_token_lost",      offsetof(totemsrp_stats_t, gather_token_lost),      ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "commit_entered",         offsetof(totemsrp_stats_t, commit_entered),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "commit_token_lost",      offsetof(totemsrp_stats_t, commit_token_lost),      ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "recovery_entered",       offsetof(totemsrp_stats_t, recovery_entered),       ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "recovery_token_lost",    offsetof(totemsrp_stats_t, recovery_token_lost),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "consensus_timeouts",     offsetof(totemsrp_stats_t, consensus_timeouts),     ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "rx_msg_dropped",         offsetof(totemsrp_stats_t, rx_msg_dropped),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "time_since_token_last_received", offsetof(totemsrp_stats_t, time_since_token_last_received), ICMAP_VALUETYPE_UINT64},
+	{ STAT_SRP, "continuous_gather",      offsetof(totemsrp_stats_t, continuous_gather),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "continuous_sendmsg_failures", offsetof(totemsrp_stats_t, continuous_sendmsg_failures), ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "firewall_enabled_or_nic_failure", offsetof(totemsrp_stats_t, firewall_enabled_or_nic_failure), ICMAP_VALUETYPE_UINT8},
+	{ STAT_SRP, "mtt_rx_token",           offsetof(totemsrp_stats_t, mtt_rx_token),           ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "avg_token_workload",     offsetof(totemsrp_stats_t, avg_token_workload),     ICMAP_VALUETYPE_UINT32},
+	{ STAT_SRP, "avg_backlog_calc",       offsetof(totemsrp_stats_t, avg_backlog_calc),       ICMAP_VALUETYPE_UINT32},
+};
+
+struct cs_stats_conv cs_knet_stats[] = {
+	{ STAT_KNET, "enabled",          offsetof(struct knet_link_status, enabled),                ICMAP_VALUETYPE_UINT8},
+	{ STAT_KNET, "connected",        offsetof(struct knet_link_status, connected),              ICMAP_VALUETYPE_UINT8},
+	{ STAT_KNET, "mtu",              offsetof(struct knet_link_status, mtu),                    ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_packets",  offsetof(struct knet_link_status, stats.tx_data_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_data_packets",  offsetof(struct knet_link_status, stats.rx_data_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_data_bytes",    offsetof(struct knet_link_status, stats.tx_data_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_data_bytes",    offsetof(struct knet_link_status, stats.rx_data_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_ping_packets",  offsetof(struct knet_link_status, stats.tx_ping_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_ping_packets",  offsetof(struct knet_link_status, stats.rx_ping_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_ping_bytes",    offsetof(struct knet_link_status, stats.tx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_ping_bytes",    offsetof(struct knet_link_status, stats.rx_ping_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pong_packets",  offsetof(struct knet_link_status, stats.tx_pong_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pong_packets",  offsetof(struct knet_link_status, stats.rx_pong_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pong_bytes",    offsetof(struct knet_link_status, stats.tx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pong_bytes",    offsetof(struct knet_link_status, stats.rx_pong_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_packets",  offsetof(struct knet_link_status, stats.tx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pmtu_packets",  offsetof(struct knet_link_status, stats.rx_pmtu_packets),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_bytes",    offsetof(struct knet_link_status, stats.tx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_pmtu_bytes",    offsetof(struct knet_link_status, stats.rx_pmtu_bytes),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_packets", offsetof(struct knet_link_status, stats.tx_total_packets), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_packets", offsetof(struct knet_link_status, stats.rx_total_packets), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_bytes",   offsetof(struct knet_link_status, stats.tx_total_bytes),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_bytes",   offsetof(struct knet_link_status, stats.rx_total_bytes),   ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_total_errors",  offsetof(struct knet_link_status, stats.tx_total_errors),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "rx_total_retries", offsetof(struct knet_link_status, stats.tx_total_retries), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET, "tx_pmtu_errors",   offsetof(struct knet_link_status, stats.tx_pmtu_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pmtu_retries",  offsetof(struct knet_link_status, stats.tx_pmtu_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_ping_errors",   offsetof(struct knet_link_status, stats.tx_ping_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_ping_retries",  offsetof(struct knet_link_status, stats.tx_ping_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pong_errors",   offsetof(struct knet_link_status, stats.tx_pong_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_pong_retries",  offsetof(struct knet_link_status, stats.tx_pong_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_errors",   offsetof(struct knet_link_status, stats.tx_data_errors),   ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "tx_data_retries",  offsetof(struct knet_link_status, stats.tx_data_retries),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_min",      offsetof(struct knet_link_status, stats.latency_min),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_max",      offsetof(struct knet_link_status, stats.latency_max),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_ave",      offsetof(struct knet_link_status, stats.latency_ave),      ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "latency_samples",  offsetof(struct knet_link_status, stats.latency_samples),  ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "down_count",       offsetof(struct knet_link_status, stats.down_count),       ICMAP_VALUETYPE_UINT32},
+	{ STAT_KNET, "up_count",         offsetof(struct knet_link_status, stats.up_count),         ICMAP_VALUETYPE_UINT32},
+};
+struct cs_stats_conv cs_knet_handle_stats[] = {
+	{ STAT_KNET_HANDLE, "tx_uncompressed_packets",      offsetof(struct knet_handle_stats, tx_uncompressed_packets),      ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compressed_packets",        offsetof(struct knet_handle_stats, tx_compressed_packets),        ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compressed_original_bytes", offsetof(struct knet_handle_stats, tx_compressed_original_bytes), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compressed_size_bytes",     offsetof(struct knet_handle_stats, tx_compressed_size_bytes),     ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compress_time_min",         offsetof(struct knet_handle_stats, tx_compress_time_min),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compress_time_max",         offsetof(struct knet_handle_stats, tx_compress_time_max),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_compress_time_ave",         offsetof(struct knet_handle_stats, tx_compress_time_ave),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compressed_packets",        offsetof(struct knet_handle_stats, rx_compressed_packets),        ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compressed_original_bytes", offsetof(struct knet_handle_stats, rx_compressed_original_bytes), ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compressed_size_bytes",     offsetof(struct knet_handle_stats, rx_compressed_size_bytes),     ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compress_time_min",         offsetof(struct knet_handle_stats, rx_compress_time_min),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compress_time_max",         offsetof(struct knet_handle_stats, rx_compress_time_max),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_compress_time_ave",         offsetof(struct knet_handle_stats, rx_compress_time_ave),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_crypt_time_min",            offsetof(struct knet_handle_stats, tx_crypt_time_min),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_crypt_time_max",            offsetof(struct knet_handle_stats, tx_crypt_time_max),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_crypt_time_ave",            offsetof(struct knet_handle_stats, tx_crypt_time_ave),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_crypt_byte_overhead",       offsetof(struct knet_handle_stats, tx_crypt_byte_overhead),       ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "tx_crypt_packets",             offsetof(struct knet_handle_stats, tx_crypt_packets),             ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_crypt_time_min",            offsetof(struct knet_handle_stats, rx_crypt_time_min),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_crypt_time_max",            offsetof(struct knet_handle_stats, rx_crypt_time_max),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_crypt_time_ave",            offsetof(struct knet_handle_stats, rx_crypt_time_ave),            ICMAP_VALUETYPE_UINT64},
+	{ STAT_KNET_HANDLE, "rx_crypt_packets",             offsetof(struct knet_handle_stats, rx_crypt_packets),             ICMAP_VALUETYPE_UINT64},
+};
+
+struct cs_stats_conv cs_ipcs_conn_stats[] = {
+	{ STAT_IPCSC, "queueing",        offsetof(struct ipcs_conn_stats, cnx.queuing),          ICMAP_VALUETYPE_INT32},
+	{ STAT_IPCSC, "queued",          offsetof(struct ipcs_conn_stats, cnx.queued),           ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "invalid_request", offsetof(struct ipcs_conn_stats, cnx.invalid_request),  ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "overload",        offsetof(struct ipcs_conn_stats, cnx.overload),         ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "sent",            offsetof(struct ipcs_conn_stats, cnx.sent),             ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "procname",        offsetof(struct ipcs_conn_stats, cnx.proc_name),        ICMAP_VALUETYPE_STRING},
+	{ STAT_IPCSC, "requests",        offsetof(struct ipcs_conn_stats, conn.requests),        ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "responses",       offsetof(struct ipcs_conn_stats, conn.responses),       ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "dispatched",      offsetof(struct ipcs_conn_stats, conn.events),          ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "send_retries",    offsetof(struct ipcs_conn_stats, conn.send_retries),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "recv_retries",    offsetof(struct ipcs_conn_stats, conn.recv_retries),    ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSC, "flow_control",    offsetof(struct ipcs_conn_stats, conn.flow_control_state),    ICMAP_VALUETYPE_UINT32},
+	{ STAT_IPCSC, "flow_control_count",   offsetof(struct ipcs_conn_stats, conn.flow_control_count),    ICMAP_VALUETYPE_UINT64},
+};
+struct cs_stats_conv cs_ipcs_global_stats[] = {
+	{ STAT_IPCSG, "global.active",        offsetof(struct ipcs_global_stats, active),           ICMAP_VALUETYPE_UINT64},
+	{ STAT_IPCSG, "global.closed",        offsetof(struct ipcs_global_stats, closed),           ICMAP_VALUETYPE_UINT64},
+};
+struct cs_stats_conv cs_schedmiss_stats[] = {
+	{ STAT_SCHEDMISS, "timestamp",    offsetof(struct schedmiss_entry, timestamp), ICMAP_VALUETYPE_UINT64},
+	{ STAT_SCHEDMISS, "delay",        offsetof(struct schedmiss_entry, delay),     ICMAP_VALUETYPE_FLOAT},
+};
+
+#define NUM_PG_STATS (sizeof(cs_pg_stats) / sizeof(struct cs_stats_conv))
+#define NUM_SRP_STATS (sizeof(cs_srp_stats) / sizeof(struct cs_stats_conv))
+#define NUM_KNET_STATS (sizeof(cs_knet_stats) / sizeof(struct cs_stats_conv))
+#define NUM_KNET_HANDLE_STATS (sizeof(cs_knet_handle_stats) / sizeof(struct cs_stats_conv))
+#define NUM_IPCSC_STATS (sizeof(cs_ipcs_conn_stats) / sizeof(struct cs_stats_conv))
+#define NUM_IPCSG_STATS (sizeof(cs_ipcs_global_stats) / sizeof(struct cs_stats_conv))
+
+/* What goes in the trie */
+struct stats_item {
+	char *key_name;
+	struct cs_stats_conv * cs_conv;
+};
+
+/* One of these per tracker */
+struct cs_stats_tracker
+{
+	char *key_name;
+	void *user_data;
+	int32_t events;
+	icmap_notify_fn_t notify_fn;
+	uint64_t old_value;
+	struct qb_list_head list;
+};
+QB_LIST_DECLARE (stats_tracker_list_head);
+static const struct corosync_api_v1 *api;
+
+static void stats_map_set_value(struct cs_stats_conv *conv,
+				void *stat_array,
+				void *value,
+				size_t *value_len,
+				icmap_value_types_t *type)
+{
+	if (value_len) {
+		*value_len = icmap_get_valuetype_len(conv->value_type);
+	}
+	if (type) {
+		*type = conv->value_type;
+		if ((*type == ICMAP_VALUETYPE_STRING) && value_len && stat_array) {
+			*value_len = strlen((char *)(stat_array) + conv->offset)+1;
+		}
+	}
+	if (value) {
+		assert(value_len != NULL);
+
+		memcpy(value, (char *)(stat_array) + conv->offset, *value_len);
+	}
+}
+
+static void stats_add_entry(const char *key, struct cs_stats_conv *cs_conv)
+{
+	struct stats_item *item = malloc(sizeof(struct stats_item));
+
+	if (item) {
+		item->cs_conv = cs_conv;
+		item->key_name = strdup(key);
+		qb_map_put(stats_map, item->key_name, item);
+	}
+}
+static void stats_rm_entry(const char *key)
+{
+	struct stats_item *item = qb_map_get(stats_map, key);
+
+	if (item) {
+		qb_map_rm(stats_map, item->key_name);
+		/* Structures freed in callback below */
+	}
+}
+
+static void stats_map_free_cb(uint32_t event,
+			      char* key, void* old_value,
+			      void* value, void* user_data)
+{
+	struct stats_item *item = (struct stats_item *)old_value;
+
+	if (item) {
+		free(item->key_name);
+		free(item);
+	}
+}
+
+cs_error_t stats_map_init(const struct corosync_api_v1 *corosync_api)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+	int32_t err;
+
+	api = corosync_api;
+
+	stats_map = qb_trie_create();
+	if (!stats_map) {
+		return CS_ERR_INIT;
+	}
+
+	/* Populate the static portions of the trie */
+	for (i = 0; i<NUM_PG_STATS; i++) {
+		sprintf(param, "stats.pg.%s", cs_pg_stats[i].name);
+		stats_add_entry(param, &cs_pg_stats[i]);
+	}
+	for (i = 0; i<NUM_SRP_STATS; i++) {
+		sprintf(param, "stats.srp.%s", cs_srp_stats[i].name);
+		stats_add_entry(param, &cs_srp_stats[i]);
+	}
+	for (i = 0; i<NUM_IPCSG_STATS; i++) {
+		sprintf(param, "stats.ipcs.%s", cs_ipcs_global_stats[i].name);
+		stats_add_entry(param, &cs_ipcs_global_stats[i]);
+	}
+
+	/* KNET, IPCS & SCHEDMISS stats are added when appropriate */
+
+
+	/* Call us when we can free things */
+	err = qb_map_notify_add(stats_map, NULL, stats_map_free_cb, QB_MAP_NOTIFY_FREE, NULL);
+
+	return (qb_to_cs_error(err));
+}
+
+cs_error_t stats_map_get(const char *key_name,
+			 void *value,
+			 size_t *value_len,
+			 icmap_value_types_t *type)
+{
+	struct cs_stats_conv *statinfo;
+	struct stats_item *item;
+	totempg_stats_t *pg_stats;
+	struct knet_link_status link_status;
+	struct ipcs_conn_stats ipcs_conn_stats;
+	struct ipcs_global_stats ipcs_global_stats;
+	struct knet_handle_stats knet_handle_stats;
+	int res;
+	int nodeid;
+	int link_no;
+	int service_id;
+	uint32_t pid;
+	unsigned int sm_event;
+	char *sm_type;
+	void *conn_ptr;
+
+	item = qb_map_get(stats_map, key_name);
+	if (!item) {
+		return CS_ERR_NOT_EXIST;
+	}
+
+	statinfo = item->cs_conv;
+	switch (statinfo->type) {
+		case STAT_PG:
+			pg_stats = api->totem_get_stats();
+			stats_map_set_value(statinfo, pg_stats, value, value_len, type);
+			break;
+		case STAT_SRP:
+			pg_stats = api->totem_get_stats();
+			stats_map_set_value(statinfo, pg_stats->srp, value, value_len, type);
+			break;
+		case STAT_KNET_HANDLE:
+			res = totemknet_handle_get_stats(&knet_handle_stats);
+			if (res != CS_OK) {
+				return res;
+			}
+			stats_map_set_value(statinfo, &knet_handle_stats, value, value_len, type);
+			break;
+		case STAT_KNET:
+			if (sscanf(key_name, "stats.knet.node%d.link%d", &nodeid, &link_no) != 2) {
+				return CS_ERR_NOT_EXIST;
+			}
+
+			/* Validate node & link IDs */
+			if (nodeid <= 0 || nodeid > KNET_MAX_HOST ||
+			    link_no < 0 || link_no > KNET_MAX_LINK) {
+				return CS_ERR_NOT_EXIST;
+			}
+
+			/* Always get the latest stats */
+			res = totemknet_link_get_status((knet_node_id_t)nodeid, (uint8_t)link_no, &link_status);
+			if (res != CS_OK) {
+				return CS_ERR_LIBRARY;
+			}
+			stats_map_set_value(statinfo, &link_status, value, value_len, type);
+			break;
+		case STAT_IPCSC:
+			if (sscanf(key_name, "stats.ipcs.service%d.%d.%p", &service_id, &pid, &conn_ptr) != 3) {
+				return CS_ERR_NOT_EXIST;
+			}
+			res = cs_ipcs_get_conn_stats(service_id, pid, conn_ptr, &ipcs_conn_stats);
+			if (res != CS_OK) {
+				return res;
+			}
+			stats_map_set_value(statinfo, &ipcs_conn_stats, value, value_len, type);
+			break;
+		case STAT_IPCSG:
+			cs_ipcs_get_global_stats(&ipcs_global_stats);
+			stats_map_set_value(statinfo, &ipcs_global_stats, value, value_len, type);
+			break;
+		case STAT_SCHEDMISS:
+			if (sscanf(key_name, SCHEDMISS_PREFIX ".%d", &sm_event) != 1) {
+				return CS_ERR_NOT_EXIST;
+			}
+
+			sm_type = strrchr(key_name, '.');
+			if (sm_type == NULL) {
+				return CS_ERR_NOT_EXIST;
+			}
+			sm_type++;
+
+			if (strcmp(sm_type, "timestamp") == 0) {
+				memcpy(value, &schedmiss_event[sm_event].timestamp, sizeof(uint64_t));
+				*value_len = sizeof(uint64_t);
+				*type = ICMAP_VALUETYPE_UINT64;
+			}
+			if (strcmp(sm_type, "delay") == 0) {
+				memcpy(value, &schedmiss_event[sm_event].delay, sizeof(float));
+				*value_len = sizeof(float);
+				*type = ICMAP_VALUETYPE_FLOAT;
+			}
+			break;
+		default:
+			return CS_ERR_LIBRARY;
+	}
+	return CS_OK;
+}
+
+static void schedmiss_clear_stats(void)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i=0; i<MAX_SCHEDMISS_EVENTS; i++) {
+		if (i < highest_schedmiss_event) {
+			sprintf(param, SCHEDMISS_PREFIX ".%i.timestamp", i);
+			stats_rm_entry(param);
+			sprintf(param, SCHEDMISS_PREFIX ".%i.delay", i);
+			stats_rm_entry(param);
+		}
+		schedmiss_event[i].timestamp = (uint64_t)0LL;
+		schedmiss_event[i].delay = 0.0f;
+	}
+	highest_schedmiss_event = 0;
+}
+
+/* Called from main.c */
+void stats_add_schedmiss_event(uint64_t timestamp, float delay)
+{
+	char param[ICMAP_KEYNAME_MAXLEN];
+	int i;
+
+	/* Move 'em all up */
+	for (i=MAX_SCHEDMISS_EVENTS-2; i>=0; i--) {
+		schedmiss_event[i+1].timestamp = schedmiss_event[i].timestamp;
+		schedmiss_event[i+1].delay = schedmiss_event[i].delay;
+	}
+
+	/* New entries are always at the front */
+	schedmiss_event[0].timestamp = timestamp;
+	schedmiss_event[0].delay = delay;
+
+	/* If we've not run off the end then add an entry in the trie for the new 'end' one */
+	if (highest_schedmiss_event < MAX_SCHEDMISS_EVENTS) {
+		sprintf(param, SCHEDMISS_PREFIX ".%i.timestamp", highest_schedmiss_event);
+		stats_add_entry(param, &cs_schedmiss_stats[0]);
+		sprintf(param, SCHEDMISS_PREFIX ".%i.delay", highest_schedmiss_event);
+		stats_add_entry(param, &cs_schedmiss_stats[1]);
+		highest_schedmiss_event++;
+	}
+	/* Notifications get sent by the stats_updater */
+}
+
+#define STATS_CLEAR           "stats.clear."
+#define STATS_CLEAR_KNET      "stats.clear.knet"
+#define STATS_CLEAR_IPC       "stats.clear.ipc"
+#define STATS_CLEAR_TOTEM     "stats.clear.totem"
+#define STATS_CLEAR_ALL       "stats.clear.all"
+#define STATS_CLEAR_SCHEDMISS "stats.clear.schedmiss"
+
+cs_error_t stats_map_set(const char *key_name,
+			 const void *value,
+			 size_t value_len,
+			 icmap_value_types_t type)
+{
+	int cleared = 0;
+
+	if (strncmp(key_name, STATS_CLEAR_KNET, strlen(STATS_CLEAR_KNET)) == 0) {
+		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TRANSPORT);
+		cleared = 1;
+	}
+	if (strncmp(key_name, STATS_CLEAR_IPC, strlen(STATS_CLEAR_IPC)) == 0) {
+		cs_ipcs_clear_stats();
+		cleared = 1;
+	}
+	if (strncmp(key_name, STATS_CLEAR_TOTEM, strlen(STATS_CLEAR_TOTEM)) == 0) {
+		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TOTEM);
+		cleared = 1;
+	}
+	if (strncmp(key_name, STATS_CLEAR_SCHEDMISS, strlen(STATS_CLEAR_SCHEDMISS)) == 0) {
+		schedmiss_clear_stats();
+		cleared = 1;
+	}
+	if (strncmp(key_name, STATS_CLEAR_ALL, strlen(STATS_CLEAR_ALL)) == 0) {
+		totempg_stats_clear(TOTEMPG_STATS_CLEAR_TRANSPORT | TOTEMPG_STATS_CLEAR_TOTEM);
+		cs_ipcs_clear_stats();
+		schedmiss_clear_stats();
+		cleared = 1;
+	}
+	if (!cleared) {
+		return CS_ERR_NOT_SUPPORTED;
+	}
+	return CS_OK;
+}
+
+cs_error_t stats_map_adjust_int(const char *key_name, int32_t step)
+{
+	return CS_ERR_NOT_SUPPORTED;
+}
+
+cs_error_t stats_map_delete(const char *key_name)
+{
+	return CS_ERR_NOT_SUPPORTED;
+}
+
+int stats_map_is_key_ro(const char *key_name)
+{
+	/* It's all read-only apart from the 'clear' destinations */
+	if (strncmp(key_name, STATS_CLEAR, strlen(STATS_CLEAR)) == 0) {
+		return 0;
+	} else {
+		return 1;
+	}
+}
+
+icmap_iter_t stats_map_iter_init(const char *prefix)
+{
+	return (qb_map_pref_iter_create(stats_map, prefix));
+}
+
+
+const char *stats_map_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
+{
+	const char *res;
+	struct stats_item *item;
+
+	res = qb_map_iter_next(iter, (void **)&item);
+	if (res == NULL) {
+		return (res);
+	}
+	stats_map_set_value(item->cs_conv, NULL, NULL, value_len, type);
+
+	return res;
+}
+
+void stats_map_iter_finalize(icmap_iter_t iter)
+{
+	qb_map_iter_free(iter);
+}
+
+
+void stats_trigger_trackers()
+{
+	struct cs_stats_tracker *tracker;
+	struct qb_list_head *iter;
+	cs_error_t res;
+	size_t value_len;
+	icmap_value_types_t type;
+	uint64_t value;
+	struct icmap_notify_value new_val;
+	struct icmap_notify_value old_val;
+
+	qb_list_for_each(iter, &stats_tracker_list_head) {
+
+		tracker = qb_list_entry(iter, struct cs_stats_tracker, list);
+		if (tracker->events & ICMAP_TRACK_PREFIX || !tracker->key_name ) {
+			continue;
+		}
+
+		res = stats_map_get(tracker->key_name,
+				    &value, &value_len, &type);
+
+		/* Check if it has changed */
+		if ((res == CS_OK) && (memcmp(&value, &tracker->old_value, value_len) != 0)) {
+
+			old_val.type = new_val.type = type;
+			old_val.len = new_val.len = value_len;
+			old_val.data = new_val.data = &value;
+
+			tracker->notify_fn(ICMAP_TRACK_MODIFY, tracker->key_name,
+					   old_val, new_val, tracker->user_data);
+
+			memcpy(&tracker->old_value, &value, value_len);
+		}
+	}
+}
+
+
+/* Callback from libqb when a key is added/removed */
+static void stats_map_notify_fn(uint32_t event, char *key, void *old_value, void *value, void *user_data)
+{
+	struct cs_stats_tracker *tracker = user_data;
+	struct icmap_notify_value new_val;
+	struct icmap_notify_value old_val;
+	char new_value[64];
+
+	if (value == NULL && old_value == NULL) {
+		return ;
+	}
+
+	/* Ignore schedmiss trackers as the values are read from the circular buffer */
+	if (strncmp(key, SCHEDMISS_PREFIX, strlen(SCHEDMISS_PREFIX)) == 0 ) {
+		return ;
+	}
+
+	new_val.data = new_value;
+	if (stats_map_get(key,
+			  &new_value,
+			  &new_val.len,
+			  &new_val.type) != CS_OK) {
+		log_printf(LOGSYS_LEVEL_WARNING, "get value of notified key %s failed", key);
+		return ;
+	}
+
+	/* We don't know what the old value was
+	   but as this only tracks ADD & DELETE I'm not worried
+	   about it */
+	memcpy(&old_val, &new_val, sizeof(new_val));
+
+	tracker->notify_fn(icmap_qbtt_to_tt(event),
+			   key,
+			   new_val,
+			   old_val,
+			   tracker->user_data);
+
+}
+
+cs_error_t stats_map_track_add(const char *key_name,
+			       int32_t track_type,
+			       icmap_notify_fn_t notify_fn,
+			       void *user_data,
+			       icmap_track_t *icmap_track)
+{
+	struct cs_stats_tracker *tracker;
+	size_t value_len;
+	icmap_value_types_t type;
+	cs_error_t err;
+
+	/* We can track adding or deleting a key under a prefix */
+	if ((track_type & ICMAP_TRACK_PREFIX) &&
+	    (!(track_type & ICMAP_TRACK_DELETE) ||
+	     !(track_type & ICMAP_TRACK_ADD))) {
+		return CS_ERR_NOT_SUPPORTED;
+	}
+
+	tracker = malloc(sizeof(struct cs_stats_tracker));
+	if (!tracker) {
+		return CS_ERR_NO_MEMORY;
+	}
+
+	tracker->notify_fn = notify_fn;
+	tracker->user_data = user_data;
+	tracker->events = track_type;
+	if (key_name) {
+		tracker->key_name = strdup(key_name);
+		if (!tracker->key_name) {
+			free(tracker);
+			return CS_ERR_NO_MEMORY;
+		}
+		/* Get initial value */
+		if (stats_map_get(tracker->key_name,
+				  &tracker->old_value, &value_len, &type) != CS_OK) {
+			tracker->old_value = 0ULL;
+		}
+	} else {
+		tracker->key_name = NULL;
+		tracker->old_value = 0ULL;
+	}
+
+	/* Add/delete trackers can use the qb_map tracking */
+	if ((track_type & ICMAP_TRACK_ADD) ||
+	    (track_type & ICMAP_TRACK_DELETE)) {
+		err = qb_map_notify_add(stats_map, tracker->key_name,
+					stats_map_notify_fn,
+					icmap_tt_to_qbtt(track_type),
+					tracker);
+		if (err != 0) {
+			log_printf(LOGSYS_LEVEL_ERROR, "creating stats tracker %s failed. %d\n", tracker->key_name, err);
+			free(tracker->key_name);
+			free(tracker);
+			return (qb_to_cs_error(err));
+		}
+	}
+
+	qb_list_add (&tracker->list, &stats_tracker_list_head);
+
+	*icmap_track = (icmap_track_t)tracker;
+	return CS_OK;
+}
+
+cs_error_t stats_map_track_delete(icmap_track_t icmap_track)
+{
+	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;
+	int err;
+
+	if ((tracker->events & ICMAP_TRACK_ADD) ||
+	    (tracker->events & ICMAP_TRACK_DELETE)) {
+		err = qb_map_notify_del_2(stats_map,
+					  tracker->key_name, stats_map_notify_fn,
+					  icmap_tt_to_qbtt(tracker->events), tracker);
+		if (err) {
+			log_printf(LOGSYS_LEVEL_ERROR, "deleting tracker %s failed. %d\n", tracker->key_name, err);
+		}
+	}
+
+	qb_list_del(&tracker->list);
+	free(tracker->key_name);
+	free(tracker);
+
+	return CS_OK;
+}
+
+void *stats_map_track_get_user_data(icmap_track_t icmap_track)
+{
+	struct cs_stats_tracker *tracker = (struct cs_stats_tracker *)icmap_track;
+
+	return tracker->user_data;
+}
+
+/* Called from totemknet to add/remove keys from our map */
+void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link_no)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_KNET_STATS; i++) {
+		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
+		stats_add_entry(param, &cs_knet_stats[i]);
+	}
+}
+void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link_no)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_KNET_STATS; i++) {
+		sprintf(param, "stats.knet.node%d.link%d.%s", nodeid, link_no, cs_knet_stats[i].name);
+		stats_rm_entry(param);
+	}
+}
+
+/* This is separated out from  stats_map_init() because we don't know whether
+   knet is in use until much later in the startup */
+void stats_knet_add_handle(void)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_KNET_HANDLE_STATS; i++) {
+		sprintf(param, "stats.knet.handle.%s", cs_knet_handle_stats[i].name);
+		stats_add_entry(param, &cs_knet_handle_stats[i]);
+	}
+}
+
+/* Called from ipc_glue to add/remove keys from our map */
+void stats_ipcs_add_connection(int service_id, uint32_t pid, void *ptr)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_IPCSC_STATS; i++) {
+		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
+		stats_add_entry(param, &cs_ipcs_conn_stats[i]);
+	}
+}
+void stats_ipcs_del_connection(int service_id, uint32_t pid, void *ptr)
+{
+	int i;
+	char param[ICMAP_KEYNAME_MAXLEN];
+
+	for (i = 0; i<NUM_IPCSC_STATS; i++) {
+		sprintf(param, "stats.ipcs.service%d.%d.%p.%s", service_id, pid, ptr, cs_ipcs_conn_stats[i].name);
+		stats_rm_entry(param);
+	}
+}