Adding upstream version 8.4.4.upstream/8.4.4upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1324 insertions, 0 deletions

diff --git a/zebra/zserv.c b/zebra/zserv.c
new file mode 100644
index 0000000..a99dce0
--- /dev/null
+++ b/zebra/zserv.c
@@ -0,0 +1,1324 @@
+/*
+ * Zebra API server.
+ * Portions:
+ *   Copyright (C) 1997-1999  Kunihiro Ishiguro
+ *   Copyright (C) 2015-2018  Cumulus Networks, Inc.
+ *   et al.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; see the file COPYING; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <zebra.h>
+
+/* clang-format off */
+#include <errno.h>                /* for errno */
+#include <netinet/in.h>           /* for sockaddr_in */
+#include <stdint.h>               /* for uint8_t */
+#include <stdio.h>                /* for snprintf */
+#include <sys/socket.h>           /* for sockaddr_storage, AF_UNIX, accept... */
+#include <sys/stat.h>             /* for umask, mode_t */
+#include <sys/un.h>               /* for sockaddr_un */
+#include <time.h>                 /* for NULL, tm, gmtime, time_t */
+#include <unistd.h>               /* for close, unlink, ssize_t */
+
+#include "lib/buffer.h"           /* for BUFFER_EMPTY, BUFFER_ERROR, BUFFE... */
+#include "lib/command.h"          /* for vty, install_element, CMD_SUCCESS... */
+#include "lib/hook.h"             /* for DEFINE_HOOK, DEFINE_KOOH, hook_call */
+#include "lib/linklist.h"         /* for ALL_LIST_ELEMENTS_RO, ALL_LIST_EL... */
+#include "lib/libfrr.h"           /* for frr_zclient_addr */
+#include "lib/log.h"              /* for zlog_warn, zlog_debug, safe_strerror */
+#include "lib/memory.h"           /* for MTYPE_TMP, XCALLOC, XFREE */
+#include "lib/monotime.h"         /* for monotime, ONE_DAY_SECOND, ONE_WEE... */
+#include "lib/network.h"          /* for set_nonblocking */
+#include "lib/privs.h"            /* for zebra_privs_t, ZPRIVS_LOWER, ZPRI... */
+#include "lib/route_types.h"      /* for ZEBRA_ROUTE_MAX */
+#include "lib/sockopt.h"          /* for setsockopt_so_recvbuf, setsockopt... */
+#include "lib/sockunion.h"        /* for sockopt_reuseaddr, sockopt_reuseport */
+#include "lib/stream.h"           /* for STREAM_SIZE, stream (ptr only), ... */
+#include "lib/thread.h"           /* for thread (ptr only), THREAD_ARG, ... */
+#include "lib/vrf.h"              /* for vrf_info_lookup, VRF_DEFAULT */
+#include "lib/vty.h"              /* for vty_out, vty (ptr only) */
+#include "lib/zclient.h"          /* for zmsghdr, ZEBRA_HEADER_SIZE, ZEBRA... */
+#include "lib/frr_pthread.h"      /* for frr_pthread_new, frr_pthread_stop... */
+#include "lib/frratomic.h"        /* for atomic_load_explicit, atomic_stor... */
+#include "lib/lib_errors.h"       /* for generic ferr ids */
+#include "lib/printfrr.h"         /* for string functions */
+
+#include "zebra/debug.h"          /* for various debugging macros */
+#include "zebra/rib.h"            /* for rib_score_proto */
+#include "zebra/zapi_msg.h"       /* for zserv_handle_commands */
+#include "zebra/zebra_vrf.h"      /* for zebra_vrf_lookup_by_id, zvrf */
+#include "zebra/zserv.h"          /* for zserv */
+#include "zebra/zebra_router.h"
+#include "zebra/zebra_errors.h"   /* for error messages */
+/* clang-format on */
+
+/* privileges */
+extern struct zebra_privs_t zserv_privs;
+
+/* The listener socket for clients connecting to us */
+static int zsock;
+
+/* The lock that protects access to zapi client objects */
+static pthread_mutex_t client_mutex;
+
+static struct zserv *find_client_internal(uint8_t proto,
+					  unsigned short instance,
+					  uint32_t session_id);
+
+
+/*
+ * Client thread events.
+ *
+ * These are used almost exclusively by client threads to drive their own event
+ * loops. The only exception is in zserv_client_create(), which pushes an
+ * initial ZSERV_CLIENT_READ event to start the API handler loop.
+ */
+enum zserv_client_event {
+	/* Schedule a socket read */
+	ZSERV_CLIENT_READ,
+	/* Schedule a buffer write */
+	ZSERV_CLIENT_WRITE,
+};
+
+/*
+ * Main thread events.
+ *
+ * These are used by client threads to notify the main thread about various
+ * events and to make processing requests.
+ */
+enum zserv_event {
+	/* Schedule listen job on Zebra API socket */
+	ZSERV_ACCEPT,
+	/* The calling client has packets on its input buffer */
+	ZSERV_PROCESS_MESSAGES,
+	/* The calling client wishes to be killed */
+	ZSERV_HANDLE_CLIENT_FAIL,
+};
+
+/*
+ * Zebra server event driver for all client threads.
+ *
+ * This is essentially a wrapper around thread_add_event() that centralizes
+ * those scheduling calls into one place.
+ *
+ * All calls to this function schedule an event on the pthread running the
+ * provided client.
+ *
+ * client
+ *    the client in question, and thread target
+ *
+ * event
+ *    the event to notify them about
+ */
+static void zserv_client_event(struct zserv *client,
+			       enum zserv_client_event event);
+
+/*
+ * Zebra server event driver for the main thread.
+ *
+ * This is essentially a wrapper around thread_add_event() that centralizes
+ * those scheduling calls into one place.
+ *
+ * All calls to this function schedule an event on Zebra's main pthread.
+ *
+ * client
+ *    the client in question
+ *
+ * event
+ *    the event to notify the main thread about
+ */
+static void zserv_event(struct zserv *client, enum zserv_event event);
+
+
+/* Client thread lifecycle -------------------------------------------------- */
+
+/*
+ * Log zapi message to zlog.
+ *
+ * errmsg (optional)
+ *    Debugging message
+ *
+ * msg
+ *    The message
+ *
+ * hdr (optional)
+ *    The message header
+ */
+void zserv_log_message(const char *errmsg, struct stream *msg,
+		       struct zmsghdr *hdr)
+{
+	zlog_debug("Rx'd ZAPI message");
+	if (errmsg)
+		zlog_debug("%s", errmsg);
+	if (hdr) {
+		zlog_debug(" Length: %d", hdr->length);
+		zlog_debug("Command: %s", zserv_command_string(hdr->command));
+		zlog_debug("    VRF: %u", hdr->vrf_id);
+	}
+	stream_hexdump(msg);
+}
+
+/*
+ * Gracefuly shut down a client connection.
+ *
+ * Cancel any pending tasks for the client's thread. Then schedule a task on
+ * the main thread to shut down the calling thread.
+ *
+ * It is not safe to close the client socket in this function. The socket is
+ * owned by the main thread.
+ *
+ * Must be called from the client pthread, never the main thread.
+ */
+static void zserv_client_fail(struct zserv *client)
+{
+	flog_warn(EC_ZEBRA_CLIENT_IO_ERROR,
+		  "Client '%s' encountered an error and is shutting down.",
+		  zebra_route_string(client->proto));
+
+	atomic_store_explicit(&client->pthread->running, false,
+			      memory_order_relaxed);
+
+	THREAD_OFF(client->t_read);
+	THREAD_OFF(client->t_write);
+	zserv_event(client, ZSERV_HANDLE_CLIENT_FAIL);
+}
+
+/*
+ * Write all pending messages to client socket.
+ *
+ * This function first attempts to flush any buffered data. If unsuccessful,
+ * the function reschedules itself and returns. If successful, it pops all
+ * available messages from the output queue and continues to write data
+ * directly to the socket until the socket would block. If the socket never
+ * blocks and all data is written, the function returns without rescheduling
+ * itself. If the socket ends up throwing EWOULDBLOCK, the remaining data is
+ * buffered and the function reschedules itself.
+ *
+ * The utility of the buffer is that it allows us to vastly reduce lock
+ * contention by allowing us to pop *all* messages off the output queue at once
+ * instead of locking and unlocking each time we want to pop a single message
+ * off the queue. The same thing could arguably be accomplished faster by
+ * allowing the main thread to write directly into the buffer instead of
+ * enqueuing packets onto an intermediary queue, but the intermediary queue
+ * allows us to expose information about input and output queues to the user in
+ * terms of number of packets rather than size of data.
+ */
+static void zserv_write(struct thread *thread)
+{
+	struct zserv *client = THREAD_ARG(thread);
+	struct stream *msg;
+	uint32_t wcmd = 0;
+	struct stream_fifo *cache;
+	uint64_t time_now = monotime(NULL);
+
+	/* If we have any data pending, try to flush it first */
+	switch (buffer_flush_all(client->wb, client->sock)) {
+	case BUFFER_ERROR:
+		goto zwrite_fail;
+	case BUFFER_PENDING:
+		frr_with_mutex (&client->stats_mtx) {
+			client->last_write_time = time_now;
+		}
+		zserv_client_event(client, ZSERV_CLIENT_WRITE);
+		return;
+	case BUFFER_EMPTY:
+		break;
+	}
+
+	cache = stream_fifo_new();
+
+	frr_with_mutex (&client->obuf_mtx) {
+		while (stream_fifo_head(client->obuf_fifo))
+			stream_fifo_push(cache,
+					 stream_fifo_pop(client->obuf_fifo));
+	}
+
+	if (cache->tail) {
+		msg = cache->tail;
+		stream_set_getp(msg, 0);
+		wcmd = stream_getw_from(msg, ZAPI_HEADER_CMD_LOCATION);
+	}
+
+	while (stream_fifo_head(cache)) {
+		msg = stream_fifo_pop(cache);
+		buffer_put(client->wb, STREAM_DATA(msg), stream_get_endp(msg));
+		stream_free(msg);
+	}
+
+	stream_fifo_free(cache);
+
+	/* If we have any data pending, try to flush it first */
+	switch (buffer_flush_all(client->wb, client->sock)) {
+	case BUFFER_ERROR:
+		goto zwrite_fail;
+	case BUFFER_PENDING:
+		frr_with_mutex (&client->stats_mtx) {
+			client->last_write_time = time_now;
+		}
+		zserv_client_event(client, ZSERV_CLIENT_WRITE);
+		return;
+	case BUFFER_EMPTY:
+		break;
+	}
+
+	frr_with_mutex (&client->stats_mtx) {
+		client->last_write_cmd = wcmd;
+		client->last_write_time = time_now;
+	}
+	return;
+
+zwrite_fail:
+	flog_warn(EC_ZEBRA_CLIENT_WRITE_FAILED,
+		  "%s: could not write to %s [fd = %d], closing.", __func__,
+		  zebra_route_string(client->proto), client->sock);
+	zserv_client_fail(client);
+}
+
+/*
+ * Read and process data from a client socket.
+ *
+ * The responsibilities here are to read raw data from the client socket,
+ * validate the header, encapsulate it into a single stream object, push it
+ * onto the input queue and then notify the main thread that there is new data
+ * available.
+ *
+ * This function first looks for any data in the client structure's working
+ * input buffer. If data is present, it is assumed that reading stopped in a
+ * previous invocation of this task and needs to be resumed to finish a message.
+ * Otherwise, the socket data stream is assumed to be at the beginning of a new
+ * ZAPI message (specifically at the header). The header is read and validated.
+ * If the header passed validation then the length field found in the header is
+ * used to compute the total length of the message. That much data is read (but
+ * not inspected), appended to the header, placed into a stream and pushed onto
+ * the client's input queue. A task is then scheduled on the main thread to
+ * process the client's input queue. Finally, if all of this was successful,
+ * this task reschedules itself.
+ *
+ * Any failure in any of these actions is handled by terminating the client.
+ */
+static void zserv_read(struct thread *thread)
+{
+	struct zserv *client = THREAD_ARG(thread);
+	int sock;
+	size_t already;
+	struct stream_fifo *cache;
+	uint32_t p2p_orig;
+
+	uint32_t p2p;
+	struct zmsghdr hdr;
+
+	p2p_orig = atomic_load_explicit(&zrouter.packets_to_process,
+					memory_order_relaxed);
+	cache = stream_fifo_new();
+	p2p = p2p_orig;
+	sock = THREAD_FD(thread);
+
+	while (p2p) {
+		ssize_t nb;
+		bool hdrvalid;
+		char errmsg[256];
+
+		already = stream_get_endp(client->ibuf_work);
+
+		/* Read length and command (if we don't have it already). */
+		if (already < ZEBRA_HEADER_SIZE) {
+			nb = stream_read_try(client->ibuf_work, sock,
+					     ZEBRA_HEADER_SIZE - already);
+			if ((nb == 0 || nb == -1)) {
+				if (IS_ZEBRA_DEBUG_EVENT)
+					zlog_debug("connection closed socket [%d]",
+						   sock);
+				goto zread_fail;
+			}
+			if (nb != (ssize_t)(ZEBRA_HEADER_SIZE - already)) {
+				/* Try again later. */
+				break;
+			}
+			already = ZEBRA_HEADER_SIZE;
+		}
+
+		/* Reset to read from the beginning of the incoming packet. */
+		stream_set_getp(client->ibuf_work, 0);
+
+		/* Fetch header values */
+		hdrvalid = zapi_parse_header(client->ibuf_work, &hdr);
+
+		if (!hdrvalid) {
+			snprintf(errmsg, sizeof(errmsg),
+				 "%s: Message has corrupt header", __func__);
+			zserv_log_message(errmsg, client->ibuf_work, NULL);
+			goto zread_fail;
+		}
+
+		/* Validate header */
+		if (hdr.marker != ZEBRA_HEADER_MARKER
+		    || hdr.version != ZSERV_VERSION) {
+			snprintf(
+				errmsg, sizeof(errmsg),
+				"Message has corrupt header\n%s: socket %d version mismatch, marker %d, version %d",
+				__func__, sock, hdr.marker, hdr.version);
+			zserv_log_message(errmsg, client->ibuf_work, &hdr);
+			goto zread_fail;
+		}
+		if (hdr.length < ZEBRA_HEADER_SIZE) {
+			snprintf(
+				errmsg, sizeof(errmsg),
+				"Message has corrupt header\n%s: socket %d message length %u is less than header size %d",
+				__func__, sock, hdr.length, ZEBRA_HEADER_SIZE);
+			zserv_log_message(errmsg, client->ibuf_work, &hdr);
+			goto zread_fail;
+		}
+		if (hdr.length > STREAM_SIZE(client->ibuf_work)) {
+			snprintf(
+				errmsg, sizeof(errmsg),
+				"Message has corrupt header\n%s: socket %d message length %u exceeds buffer size %lu",
+				__func__, sock, hdr.length,
+				(unsigned long)STREAM_SIZE(client->ibuf_work));
+			zserv_log_message(errmsg, client->ibuf_work, &hdr);
+			goto zread_fail;
+		}
+
+		/* Read rest of data. */
+		if (already < hdr.length) {
+			nb = stream_read_try(client->ibuf_work, sock,
+					     hdr.length - already);
+			if ((nb == 0 || nb == -1)) {
+				if (IS_ZEBRA_DEBUG_EVENT)
+					zlog_debug(
+						   "connection closed [%d] when reading zebra data",
+						   sock);
+				goto zread_fail;
+			}
+			if (nb != (ssize_t)(hdr.length - already)) {
+				/* Try again later. */
+				break;
+			}
+		}
+
+		/* Debug packet information. */
+		if (IS_ZEBRA_DEBUG_PACKET)
+			zlog_debug("zebra message[%s:%u:%u] comes from socket [%d]",
+				   zserv_command_string(hdr.command),
+				   hdr.vrf_id, hdr.length,
+				   sock);
+
+		stream_set_getp(client->ibuf_work, 0);
+		struct stream *msg = stream_dup(client->ibuf_work);
+
+		stream_fifo_push(cache, msg);
+		stream_reset(client->ibuf_work);
+		p2p--;
+	}
+
+	if (p2p < p2p_orig) {
+		uint64_t time_now = monotime(NULL);
+
+		/* update session statistics */
+		frr_with_mutex (&client->stats_mtx) {
+			client->last_read_time = time_now;
+			client->last_read_cmd = hdr.command;
+		}
+
+		/* publish read packets on client's input queue */
+		frr_with_mutex (&client->ibuf_mtx) {
+			while (cache->head)
+				stream_fifo_push(client->ibuf_fifo,
+						 stream_fifo_pop(cache));
+		}
+
+		/* Schedule job to process those packets */
+		zserv_event(client, ZSERV_PROCESS_MESSAGES);
+
+	}
+
+	if (IS_ZEBRA_DEBUG_PACKET)
+		zlog_debug("Read %d packets from client: %s", p2p_orig - p2p,
+			   zebra_route_string(client->proto));
+
+	/* Reschedule ourselves */
+	zserv_client_event(client, ZSERV_CLIENT_READ);
+
+	stream_fifo_free(cache);
+
+	return;
+
+zread_fail:
+	stream_fifo_free(cache);
+	zserv_client_fail(client);
+}
+
+static void zserv_client_event(struct zserv *client,
+			       enum zserv_client_event event)
+{
+	switch (event) {
+	case ZSERV_CLIENT_READ:
+		thread_add_read(client->pthread->master, zserv_read, client,
+				client->sock, &client->t_read);
+		break;
+	case ZSERV_CLIENT_WRITE:
+		thread_add_write(client->pthread->master, zserv_write, client,
+				 client->sock, &client->t_write);
+		break;
+	}
+}
+
+/* Main thread lifecycle ---------------------------------------------------- */
+
+/*
+ * Read and process messages from a client.
+ *
+ * This task runs on the main pthread. It is scheduled by client pthreads when
+ * they have new messages available on their input queues. The client is passed
+ * as the task argument.
+ *
+ * Each message is popped off the client's input queue and the action associated
+ * with the message is executed. This proceeds until there are no more messages,
+ * an error occurs, or the processing limit is reached.
+ *
+ * The client's I/O thread can push at most zrouter.packets_to_process messages
+ * onto the input buffer before notifying us there are packets to read. As long
+ * as we always process zrouter.packets_to_process messages here, then we can
+ * rely on the read thread to handle queuing this task enough times to process
+ * everything on the input queue.
+ */
+static void zserv_process_messages(struct thread *thread)
+{
+	struct zserv *client = THREAD_ARG(thread);
+	struct stream *msg;
+	struct stream_fifo *cache = stream_fifo_new();
+	uint32_t p2p = zrouter.packets_to_process;
+	bool need_resched = false;
+
+	frr_with_mutex (&client->ibuf_mtx) {
+		uint32_t i;
+		for (i = 0; i < p2p && stream_fifo_head(client->ibuf_fifo);
+		     ++i) {
+			msg = stream_fifo_pop(client->ibuf_fifo);
+			stream_fifo_push(cache, msg);
+		}
+
+		msg = NULL;
+
+		/* Need to reschedule processing work if there are still
+		 * packets in the fifo.
+		 */
+		if (stream_fifo_head(client->ibuf_fifo))
+			need_resched = true;
+	}
+
+	/* Process the batch of messages */
+	if (stream_fifo_head(cache))
+		zserv_handle_commands(client, cache);
+
+	stream_fifo_free(cache);
+
+	/* Reschedule ourselves if necessary */
+	if (need_resched)
+		zserv_event(client, ZSERV_PROCESS_MESSAGES);
+}
+
+int zserv_send_message(struct zserv *client, struct stream *msg)
+{
+	frr_with_mutex (&client->obuf_mtx) {
+		stream_fifo_push(client->obuf_fifo, msg);
+	}
+
+	zserv_client_event(client, ZSERV_CLIENT_WRITE);
+
+	return 0;
+}
+
+/*
+ * Send a batch of messages to a connected Zebra API client.
+ */
+int zserv_send_batch(struct zserv *client, struct stream_fifo *fifo)
+{
+	struct stream *msg;
+
+	frr_with_mutex (&client->obuf_mtx) {
+		msg = stream_fifo_pop(fifo);
+		while (msg) {
+			stream_fifo_push(client->obuf_fifo, msg);
+			msg = stream_fifo_pop(fifo);
+		}
+	}
+
+	zserv_client_event(client, ZSERV_CLIENT_WRITE);
+
+	return 0;
+}
+
+/* Hooks for client connect / disconnect */
+DEFINE_HOOK(zserv_client_connect, (struct zserv *client), (client));
+DEFINE_KOOH(zserv_client_close, (struct zserv *client), (client));
+
+/*
+ * Deinitialize zebra client.
+ *
+ * - Deregister and deinitialize related internal resources
+ * - Gracefuly close socket
+ * - Free associated resources
+ * - Free client structure
+ *
+ * This does *not* take any action on the struct thread * fields. These are
+ * managed by the owning pthread and any tasks associated with them must have
+ * been stopped prior to invoking this function.
+ */
+static void zserv_client_free(struct zserv *client)
+{
+	if (client == NULL)
+		return;
+
+	hook_call(zserv_client_close, client);
+
+	/* Close file descriptor. */
+	if (client->sock) {
+		unsigned long nroutes;
+		unsigned long nnhgs;
+
+		close(client->sock);
+
+		if (DYNAMIC_CLIENT_GR_DISABLED(client)) {
+			zebra_mpls_client_cleanup_vrf_label(client->proto);
+
+			nroutes = rib_score_proto(client->proto,
+						  client->instance);
+			zlog_notice(
+				"client %d disconnected %lu %s routes removed from the rib",
+				client->sock, nroutes,
+				zebra_route_string(client->proto));
+
+			/* Not worrying about instance for now */
+			nnhgs = zebra_nhg_score_proto(client->proto);
+			zlog_notice(
+				"client %d disconnected %lu %s nhgs removed from the rib",
+				client->sock, nnhgs,
+				zebra_route_string(client->proto));
+		}
+		client->sock = -1;
+	}
+
+	/* Free stream buffers. */
+	if (client->ibuf_work)
+		stream_free(client->ibuf_work);
+	if (client->obuf_work)
+		stream_free(client->obuf_work);
+	if (client->ibuf_fifo)
+		stream_fifo_free(client->ibuf_fifo);
+	if (client->obuf_fifo)
+		stream_fifo_free(client->obuf_fifo);
+	if (client->wb)
+		buffer_free(client->wb);
+
+	/* Free buffer mutexes */
+	pthread_mutex_destroy(&client->stats_mtx);
+	pthread_mutex_destroy(&client->obuf_mtx);
+	pthread_mutex_destroy(&client->ibuf_mtx);
+
+	/* Free bitmaps. */
+	for (afi_t afi = AFI_IP; afi < AFI_MAX; afi++) {
+		for (int i = 0; i < ZEBRA_ROUTE_MAX; i++) {
+			vrf_bitmap_free(client->redist[afi][i]);
+			redist_del_all_instances(&client->mi_redist[afi][i]);
+		}
+
+		vrf_bitmap_free(client->redist_default[afi]);
+		vrf_bitmap_free(client->ridinfo[afi]);
+		vrf_bitmap_free(client->nhrp_neighinfo[afi]);
+	}
+
+	/*
+	 * If any instance are graceful restart enabled,
+	 * client is not deleted
+	 */
+	if (DYNAMIC_CLIENT_GR_DISABLED(client)) {
+		if (IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("%s: Deleting client %s", __func__,
+				   zebra_route_string(client->proto));
+		XFREE(MTYPE_TMP, client);
+	} else {
+		/* Handle cases where client has GR instance. */
+		if (IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("%s: client %s restart enabled", __func__,
+				   zebra_route_string(client->proto));
+		if (zebra_gr_client_disconnect(client) < 0)
+			zlog_err(
+				"%s: GR enabled but could not handle disconnect event",
+				__func__);
+	}
+}
+
+void zserv_close_client(struct zserv *client)
+{
+	bool free_p = true;
+
+	if (client->pthread) {
+		/* synchronously stop and join pthread */
+		frr_pthread_stop(client->pthread, NULL);
+
+		if (IS_ZEBRA_DEBUG_EVENT)
+			zlog_debug("Closing client '%s'",
+				   zebra_route_string(client->proto));
+
+		thread_cancel_event(zrouter.master, client);
+		THREAD_OFF(client->t_cleanup);
+		THREAD_OFF(client->t_process);
+
+		/* destroy pthread */
+		frr_pthread_destroy(client->pthread);
+		client->pthread = NULL;
+	}
+
+	/*
+	 * Final check in case the client struct is in use in another
+	 * pthread: if not in-use, continue and free the client
+	 */
+	frr_with_mutex (&client_mutex) {
+		if (client->busy_count <= 0) {
+			/* remove from client list */
+			listnode_delete(zrouter.client_list, client);
+		} else {
+			/*
+			 * The client session object may be in use, although
+			 * the associated pthread is gone. Defer final
+			 * cleanup.
+			 */
+			client->is_closed = true;
+			free_p = false;
+		}
+	}
+
+	/* delete client */
+	if (free_p)
+		zserv_client_free(client);
+}
+
+/*
+ * This task is scheduled by a ZAPI client pthread on the main pthread when it
+ * wants to stop itself. When this executes, the client connection should
+ * already have been closed and the thread will most likely have died, but its
+ * resources still need to be cleaned up.
+ */
+static void zserv_handle_client_fail(struct thread *thread)
+{
+	struct zserv *client = THREAD_ARG(thread);
+
+	zserv_close_client(client);
+}
+
+/*
+ * Create a new client.
+ *
+ * This is called when a new connection is accept()'d on the ZAPI socket. It
+ * initializes new client structure, notifies any subscribers of the connection
+ * event and spawns the client's thread.
+ *
+ * sock
+ *    client's socket file descriptor
+ */
+static struct zserv *zserv_client_create(int sock)
+{
+	struct zserv *client;
+	size_t stream_size =
+		MAX(ZEBRA_MAX_PACKET_SIZ, sizeof(struct zapi_route));
+	int i;
+	afi_t afi;
+
+	client = XCALLOC(MTYPE_TMP, sizeof(struct zserv));
+
+	/* Make client input/output buffer. */
+	client->sock = sock;
+	client->ibuf_fifo = stream_fifo_new();
+	client->obuf_fifo = stream_fifo_new();
+	client->ibuf_work = stream_new(stream_size);
+	client->obuf_work = stream_new(stream_size);
+	client->connect_time = monotime(NULL);
+	pthread_mutex_init(&client->ibuf_mtx, NULL);
+	pthread_mutex_init(&client->obuf_mtx, NULL);
+	pthread_mutex_init(&client->stats_mtx, NULL);
+	client->wb = buffer_new(0);
+	TAILQ_INIT(&(client->gr_info_queue));
+
+	/* Initialize flags */
+	for (afi = AFI_IP; afi < AFI_MAX; afi++) {
+		for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
+			client->redist[afi][i] = vrf_bitmap_init();
+		client->redist_default[afi] = vrf_bitmap_init();
+		client->ridinfo[afi] = vrf_bitmap_init();
+		client->nhrp_neighinfo[afi] = vrf_bitmap_init();
+	}
+
+	/* Add this client to linked list. */
+	frr_with_mutex (&client_mutex) {
+		listnode_add(zrouter.client_list, client);
+	}
+
+	struct frr_pthread_attr zclient_pthr_attrs = {
+		.start = frr_pthread_attr_default.start,
+		.stop = frr_pthread_attr_default.stop
+	};
+	client->pthread =
+		frr_pthread_new(&zclient_pthr_attrs, "Zebra API client thread",
+				"zebra_apic");
+
+	/* start read loop */
+	zserv_client_event(client, ZSERV_CLIENT_READ);
+
+	/* call callbacks */
+	hook_call(zserv_client_connect, client);
+
+	/* start pthread */
+	frr_pthread_run(client->pthread, NULL);
+
+	return client;
+}
+
+/*
+ * Retrieve a client object by the complete tuple of
+ * {protocol, instance, session}. This version supports use
+ * from a different pthread: the object will be returned marked
+ * in-use. The caller *must* release the client object with the
+ * release_client() api, to ensure that the in-use marker is cleared properly.
+ */
+struct zserv *zserv_acquire_client(uint8_t proto, unsigned short instance,
+				   uint32_t session_id)
+{
+	struct zserv *client = NULL;
+
+	frr_with_mutex (&client_mutex) {
+		client = find_client_internal(proto, instance, session_id);
+		if (client) {
+			/* Don't return a dead/closed client object */
+			if (client->is_closed)
+				client = NULL;
+			else
+				client->busy_count++;
+		}
+	}
+
+	return client;
+}
+
+/*
+ * Release a client object that was acquired with the acquire_client() api.
+ * After this has been called, the caller must not use the client pointer -
+ * it may be freed if the client has closed.
+ */
+void zserv_release_client(struct zserv *client)
+{
+	/*
+	 * Once we've decremented the client object's refcount, it's possible
+	 * for it to be deleted as soon as we release the lock, so we won't
+	 * touch the object again.
+	 */
+	frr_with_mutex (&client_mutex) {
+		client->busy_count--;
+
+		if (client->busy_count <= 0) {
+			/*
+			 * No more users of the client object. If the client
+			 * session is closed, schedule cleanup on the zebra
+			 * main pthread.
+			 */
+			if (client->is_closed)
+				thread_add_event(zrouter.master,
+						 zserv_handle_client_fail,
+						 client, 0, &client->t_cleanup);
+		}
+	}
+
+	/*
+	 * Cleanup must take place on the zebra main pthread, so we've
+	 * scheduled an event.
+	 */
+}
+
+/*
+ * Accept socket connection.
+ */
+static void zserv_accept(struct thread *thread)
+{
+	int accept_sock;
+	int client_sock;
+	struct sockaddr_in client;
+	socklen_t len;
+
+	accept_sock = THREAD_FD(thread);
+
+	/* Reregister myself. */
+	zserv_event(NULL, ZSERV_ACCEPT);
+
+	len = sizeof(struct sockaddr_in);
+	client_sock = accept(accept_sock, (struct sockaddr *)&client, &len);
+
+	if (client_sock < 0) {
+		flog_err_sys(EC_LIB_SOCKET, "Can't accept zebra socket: %s",
+			     safe_strerror(errno));
+		return;
+	}
+
+	/* Make client socket non-blocking.  */
+	set_nonblocking(client_sock);
+
+	/* Create new zebra client. */
+	zserv_client_create(client_sock);
+}
+
+void zserv_close(void)
+{
+	/*
+	 * On shutdown, let's close the socket down
+	 * so that long running processes of killing the
+	 * routing table doesn't leave us in a bad
+	 * state where a client tries to reconnect
+	 */
+	close(zsock);
+	zsock = -1;
+
+	/* Free client list's mutex */
+	pthread_mutex_destroy(&client_mutex);
+}
+
+void zserv_start(char *path)
+{
+	int ret;
+	mode_t old_mask;
+	struct sockaddr_storage sa;
+	socklen_t sa_len;
+
+	if (!frr_zclient_addr(&sa, &sa_len, path))
+		/* should be caught in zebra main() */
+		return;
+
+	/* Set umask */
+	old_mask = umask(0077);
+
+	/* Make UNIX domain socket. */
+	zsock = socket(sa.ss_family, SOCK_STREAM, 0);
+	if (zsock < 0) {
+		flog_err_sys(EC_LIB_SOCKET, "Can't create zserv socket: %s",
+			     safe_strerror(errno));
+		return;
+	}
+
+	if (sa.ss_family != AF_UNIX) {
+		sockopt_reuseaddr(zsock);
+		sockopt_reuseport(zsock);
+	} else {
+		struct sockaddr_un *suna = (struct sockaddr_un *)&sa;
+		if (suna->sun_path[0])
+			unlink(suna->sun_path);
+	}
+
+	setsockopt_so_recvbuf(zsock, 1048576);
+	setsockopt_so_sendbuf(zsock, 1048576);
+
+	frr_with_privs((sa.ss_family != AF_UNIX) ? &zserv_privs : NULL) {
+		ret = bind(zsock, (struct sockaddr *)&sa, sa_len);
+	}
+	if (ret < 0) {
+		flog_err_sys(EC_LIB_SOCKET, "Can't bind zserv socket on %s: %s",
+			     path, safe_strerror(errno));
+		close(zsock);
+		zsock = -1;
+		return;
+	}
+
+	ret = listen(zsock, 5);
+	if (ret < 0) {
+		flog_err_sys(EC_LIB_SOCKET,
+			     "Can't listen to zserv socket %s: %s", path,
+			     safe_strerror(errno));
+		close(zsock);
+		zsock = -1;
+		return;
+	}
+
+	umask(old_mask);
+
+	zserv_event(NULL, ZSERV_ACCEPT);
+}
+
+void zserv_event(struct zserv *client, enum zserv_event event)
+{
+	switch (event) {
+	case ZSERV_ACCEPT:
+		thread_add_read(zrouter.master, zserv_accept, NULL, zsock,
+				NULL);
+		break;
+	case ZSERV_PROCESS_MESSAGES:
+		thread_add_event(zrouter.master, zserv_process_messages, client,
+				 0, &client->t_process);
+		break;
+	case ZSERV_HANDLE_CLIENT_FAIL:
+		thread_add_event(zrouter.master, zserv_handle_client_fail,
+				 client, 0, &client->t_cleanup);
+	}
+}
+
+
+/* General purpose ---------------------------------------------------------- */
+
+#define ZEBRA_TIME_BUF 32
+static char *zserv_time_buf(time_t *time1, char *buf, int buflen)
+{
+	time_t now;
+
+	assert(buf != NULL);
+	assert(buflen >= ZEBRA_TIME_BUF);
+	assert(time1 != NULL);
+
+	if (!*time1) {
+		snprintf(buf, buflen, "never   ");
+		return (buf);
+	}
+
+	now = monotime(NULL);
+	now -= *time1;
+
+	frrtime_to_interval(now, buf, buflen);
+
+	return buf;
+}
+
+/* Display client info details */
+static void zebra_show_client_detail(struct vty *vty, struct zserv *client)
+{
+	char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF];
+	char wbuf[ZEBRA_TIME_BUF], nhbuf[ZEBRA_TIME_BUF], mbuf[ZEBRA_TIME_BUF];
+	time_t connect_time, last_read_time, last_write_time;
+	uint32_t last_read_cmd, last_write_cmd;
+
+	vty_out(vty, "Client: %s", zebra_route_string(client->proto));
+	if (client->instance)
+		vty_out(vty, " Instance: %u", client->instance);
+	if (client->session_id)
+		vty_out(vty, " [%u]", client->session_id);
+	vty_out(vty, "\n");
+
+	vty_out(vty, "------------------------ \n");
+	vty_out(vty, "FD: %d \n", client->sock);
+
+	frr_with_mutex (&client->stats_mtx) {
+		connect_time = client->connect_time;
+		last_read_time = client->last_read_time;
+		last_write_time = client->last_write_time;
+
+		last_read_cmd = client->last_read_cmd;
+		last_write_cmd = client->last_write_cmd;
+	}
+
+	vty_out(vty, "Connect Time: %s \n",
+		zserv_time_buf(&connect_time, cbuf, ZEBRA_TIME_BUF));
+	if (client->nh_reg_time) {
+		vty_out(vty, "Nexthop Registry Time: %s \n",
+			zserv_time_buf(&client->nh_reg_time, nhbuf,
+				       ZEBRA_TIME_BUF));
+		if (client->nh_last_upd_time)
+			vty_out(vty, "Nexthop Last Update Time: %s \n",
+				zserv_time_buf(&client->nh_last_upd_time, mbuf,
+					       ZEBRA_TIME_BUF));
+		else
+			vty_out(vty, "No Nexthop Update sent\n");
+	} else
+		vty_out(vty, "Not registered for Nexthop Updates\n");
+
+	vty_out(vty, "Client will %sbe notified about it's routes status\n",
+		client->notify_owner ? "" : "Not ");
+
+	vty_out(vty, "Last Msg Rx Time: %s \n",
+		zserv_time_buf(&last_read_time, rbuf, ZEBRA_TIME_BUF));
+	vty_out(vty, "Last Msg Tx Time: %s \n",
+		zserv_time_buf(&last_write_time, wbuf, ZEBRA_TIME_BUF));
+	if (last_read_cmd)
+		vty_out(vty, "Last Rcvd Cmd: %s \n",
+			zserv_command_string(last_read_cmd));
+	if (last_write_cmd)
+		vty_out(vty, "Last Sent Cmd: %s \n",
+			zserv_command_string(last_write_cmd));
+	vty_out(vty, "\n");
+
+	vty_out(vty, "Type        Add         Update      Del \n");
+	vty_out(vty, "================================================== \n");
+	vty_out(vty, "IPv4        %-12u%-12u%-12u\n", client->v4_route_add_cnt,
+		client->v4_route_upd8_cnt, client->v4_route_del_cnt);
+	vty_out(vty, "IPv6        %-12u%-12u%-12u\n", client->v6_route_add_cnt,
+		client->v6_route_upd8_cnt, client->v6_route_del_cnt);
+	vty_out(vty, "Redist:v4   %-12u%-12u%-12u\n", client->redist_v4_add_cnt,
+		0, client->redist_v4_del_cnt);
+	vty_out(vty, "Redist:v6   %-12u%-12u%-12u\n", client->redist_v6_add_cnt,
+		0, client->redist_v6_del_cnt);
+	vty_out(vty, "VRF         %-12u%-12u%-12u\n", client->vrfadd_cnt, 0,
+		client->vrfdel_cnt);
+	vty_out(vty, "Connected   %-12u%-12u%-12u\n", client->ifadd_cnt, 0,
+		client->ifdel_cnt);
+	vty_out(vty, "Interface   %-12u%-12u%-12u\n", client->ifup_cnt, 0,
+		client->ifdown_cnt);
+	vty_out(vty, "Intf Addr   %-12u%-12u%-12u\n",
+		client->connected_rt_add_cnt, 0, client->connected_rt_del_cnt);
+	vty_out(vty, "BFD peer    %-12u%-12u%-12u\n", client->bfd_peer_add_cnt,
+		client->bfd_peer_upd8_cnt, client->bfd_peer_del_cnt);
+	vty_out(vty, "NHT v4      %-12u%-12u%-12u\n",
+		client->v4_nh_watch_add_cnt, 0, client->v4_nh_watch_rem_cnt);
+	vty_out(vty, "NHT v6      %-12u%-12u%-12u\n",
+		client->v6_nh_watch_add_cnt, 0, client->v6_nh_watch_rem_cnt);
+	vty_out(vty, "VxLAN SG    %-12u%-12u%-12u\n", client->vxlan_sg_add_cnt,
+		0, client->vxlan_sg_del_cnt);
+	vty_out(vty, "VNI         %-12u%-12u%-12u\n", client->vniadd_cnt, 0,
+		client->vnidel_cnt);
+	vty_out(vty, "L3-VNI      %-12u%-12u%-12u\n", client->l3vniadd_cnt, 0,
+		client->l3vnidel_cnt);
+	vty_out(vty, "MAC-IP      %-12u%-12u%-12u\n", client->macipadd_cnt, 0,
+		client->macipdel_cnt);
+	vty_out(vty, "ES          %-12u%-12u%-12u\n", client->local_es_add_cnt,
+		0, client->local_es_del_cnt);
+	vty_out(vty, "ES-EVI      %-12u%-12u%-12u\n",
+		client->local_es_evi_add_cnt, 0, client->local_es_evi_del_cnt);
+	vty_out(vty, "Errors: %u\n", client->error_cnt);
+
+#if defined DEV_BUILD
+	vty_out(vty, "Input Fifo: %zu:%zu Output Fifo: %zu:%zu\n",
+		client->ibuf_fifo->count, client->ibuf_fifo->max_count,
+		client->obuf_fifo->count, client->obuf_fifo->max_count);
+#endif
+	vty_out(vty, "\n");
+}
+
+/* Display stale client information */
+static void zebra_show_stale_client_detail(struct vty *vty,
+					   struct zserv *client)
+{
+	char buf[PREFIX2STR_BUFFER];
+	time_t uptime;
+	struct client_gr_info *info = NULL;
+	struct zserv *s = NULL;
+	bool first_p = true;
+
+	TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) {
+		if (first_p) {
+			vty_out(vty, "Stale Client Information\n");
+			vty_out(vty, "------------------------\n");
+
+			if (client->instance)
+				vty_out(vty, " Instance: %u", client->instance);
+			if (client->session_id)
+				vty_out(vty, " [%u]", client->session_id);
+
+			first_p = false;
+		}
+
+		vty_out(vty, "VRF : %s\n", vrf_id_to_name(info->vrf_id));
+		vty_out(vty, "Capabilities : ");
+		switch (info->capabilities) {
+		case ZEBRA_CLIENT_GR_CAPABILITIES:
+			vty_out(vty, "Graceful Restart(%u seconds)\n",
+				info->stale_removal_time);
+			break;
+		case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE:
+		case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING:
+		case ZEBRA_CLIENT_GR_DISABLE:
+		case ZEBRA_CLIENT_RIB_STALE_TIME:
+			vty_out(vty, "None\n");
+			break;
+		}
+
+		if (ZEBRA_CLIENT_GR_ENABLED(info->capabilities)) {
+			if (info->stale_client_ptr) {
+				s = (struct zserv *)(info->stale_client_ptr);
+				uptime = monotime(NULL);
+				uptime -= s->restart_time;
+
+				frrtime_to_interval(uptime, buf, sizeof(buf));
+
+				vty_out(vty, "Last restart time : %s ago\n",
+					buf);
+
+				vty_out(vty, "Stalepath removal time: %d sec\n",
+					info->stale_removal_time);
+				if (info->t_stale_removal) {
+					vty_out(vty,
+						"Stale delete timer: %ld sec\n",
+						thread_timer_remain_second(
+							info->t_stale_removal));
+				}
+			}
+			vty_out(vty, "Current AFI : %d\n", info->current_afi);
+			if (info->current_prefix)
+				vty_out(vty, "Current prefix : %pFX\n",
+					info->current_prefix);
+		}
+	}
+	vty_out(vty, "\n");
+	return;
+}
+
+static void zebra_show_client_brief(struct vty *vty, struct zserv *client)
+{
+	char client_string[80];
+	char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF];
+	char wbuf[ZEBRA_TIME_BUF];
+	time_t connect_time, last_read_time, last_write_time;
+
+	frr_with_mutex (&client->stats_mtx) {
+		connect_time = client->connect_time;
+		last_read_time = client->last_read_time;
+		last_write_time = client->last_write_time;
+	}
+
+	if (client->instance || client->session_id)
+		snprintfrr(client_string, sizeof(client_string), "%s[%u:%u]",
+			   zebra_route_string(client->proto), client->instance,
+			   client->session_id);
+	else
+		snprintfrr(client_string, sizeof(client_string), "%s",
+			   zebra_route_string(client->proto));
+
+	vty_out(vty, "%-10s%12s %12s%12s %10d/%-10d %10d/%-10d\n",
+		client_string,
+		zserv_time_buf(&connect_time, cbuf, ZEBRA_TIME_BUF),
+		zserv_time_buf(&last_read_time, rbuf, ZEBRA_TIME_BUF),
+		zserv_time_buf(&last_write_time, wbuf, ZEBRA_TIME_BUF),
+		client->v4_route_add_cnt + client->v4_route_upd8_cnt,
+		client->v4_route_del_cnt,
+		client->v6_route_add_cnt + client->v6_route_upd8_cnt,
+		client->v6_route_del_cnt);
+}
+
+/*
+ * Common logic that searches the client list for a zapi client; this
+ * MUST be called holding the client list mutex.
+ */
+static struct zserv *find_client_internal(uint8_t proto,
+					  unsigned short instance,
+					  uint32_t session_id)
+{
+	struct listnode *node, *nnode;
+	struct zserv *client = NULL;
+
+	for (ALL_LIST_ELEMENTS(zrouter.client_list, node, nnode, client)) {
+		if (client->proto == proto && client->instance == instance &&
+		    client->session_id == session_id)
+			break;
+	}
+
+	return client;
+}
+
+/*
+ * Public api that searches for a client session; this version is
+ * used from the zebra main pthread.
+ */
+struct zserv *zserv_find_client(uint8_t proto, unsigned short instance)
+{
+	struct zserv *client;
+
+	frr_with_mutex (&client_mutex) {
+		client = find_client_internal(proto, instance, 0);
+	}
+
+	return client;
+}
+
+/*
+ * Retrieve a client by its protocol, instance number, and session id.
+ */
+struct zserv *zserv_find_client_session(uint8_t proto, unsigned short instance,
+					uint32_t session_id)
+{
+	struct zserv *client;
+
+	frr_with_mutex (&client_mutex) {
+		client = find_client_internal(proto, instance, session_id);
+	}
+
+	return client;
+
+}
+
+/* This command is for debugging purpose. */
+DEFUN (show_zebra_client,
+       show_zebra_client_cmd,
+       "show zebra client",
+       SHOW_STR
+       ZEBRA_STR
+       "Client information\n")
+{
+	struct listnode *node;
+	struct zserv *client;
+
+	for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client)) {
+		zebra_show_client_detail(vty, client);
+		/* Show GR info if present */
+		zebra_show_stale_client_detail(vty, client);
+	}
+
+	return CMD_SUCCESS;
+}
+
+/* This command is for debugging purpose. */
+DEFUN (show_zebra_client_summary,
+       show_zebra_client_summary_cmd,
+       "show zebra client summary",
+       SHOW_STR
+       ZEBRA_STR
+       "Client information brief\n"
+       "Brief Summary\n")
+{
+	struct listnode *node;
+	struct zserv *client;
+
+	vty_out(vty,
+		"Name      Connect Time    Last Read  Last Write      IPv4 Routes           IPv6 Routes\n");
+	vty_out(vty,
+		"------------------------------------------------------------------------------------------\n");
+
+	for (ALL_LIST_ELEMENTS_RO(zrouter.client_list, node, client))
+		zebra_show_client_brief(vty, client);
+
+	vty_out(vty, "Routes column shows (added+updated)/deleted\n");
+	return CMD_SUCCESS;
+}
+
+static int zserv_client_close_cb(struct zserv *closed_client)
+{
+	struct listnode *node, *nnode;
+	struct zserv *client = NULL;
+
+	for (ALL_LIST_ELEMENTS(zrouter.client_list, node, nnode, client)) {
+		if (client->proto == closed_client->proto)
+			continue;
+
+		zsend_client_close_notify(client, closed_client);
+	}
+
+	return 0;
+}
+
+void zserv_init(void)
+{
+	/* Client list init. */
+	zrouter.client_list = list_new();
+	zrouter.stale_client_list = list_new();
+
+	/* Misc init. */
+	zsock = -1;
+	pthread_mutex_init(&client_mutex, NULL);
+
+	install_element(ENABLE_NODE, &show_zebra_client_cmd);
+	install_element(ENABLE_NODE, &show_zebra_client_summary_cmd);
+
+	hook_register(zserv_client_close, zserv_client_close_cb);
+}