/* * Copyright (c) 2008-2020 Red Hat, Inc. * * All rights reserved. * * Author: Christine Caulfield (ccaulfie@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* * Provides a quorum API using the corosync executive */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "util.h" struct quorum_inst { qb_ipcc_connection_t *c; int finalize; const void *context; union { quorum_model_data_t model_data; quorum_model_v0_data_t model_v0_data; quorum_model_v1_data_t model_v1_data; }; }; static void quorum_inst_free (void *inst); DECLARE_HDB_DATABASE(quorum_handle_t_db, quorum_inst_free); cs_error_t quorum_initialize ( quorum_handle_t *handle, quorum_callbacks_t *callbacks, uint32_t *quorum_type) { quorum_model_v0_data_t model_v0_data; memset (&model_v0_data, 0, sizeof(quorum_model_v0_data_t)); if (callbacks) { model_v0_data.quorum_notify_fn = callbacks->quorum_notify_fn; } return (quorum_model_initialize(handle, QUORUM_MODEL_V0, (quorum_model_data_t *)&model_v0_data, quorum_type, NULL)); } cs_error_t quorum_model_initialize ( quorum_handle_t *handle, quorum_model_t model, quorum_model_data_t *model_data, uint32_t *quorum_type, void *context) { cs_error_t error; struct quorum_inst *quorum_inst; struct iovec iov; struct qb_ipc_request_header quorum_gettype_req; struct req_lib_quorum_model_gettype quorum_model_gettype_req; struct res_lib_quorum_gettype res_lib_quorum_gettype; struct res_lib_quorum_model_gettype res_lib_quorum_model_gettype; uint32_t local_quorum_type; if (model != QUORUM_MODEL_V0 && model != QUORUM_MODEL_V1) { error = CS_ERR_INVALID_PARAM; goto error_no_destroy; } error = hdb_error_to_cs(hdb_handle_create (&quorum_handle_t_db, sizeof (struct quorum_inst), handle)); if (error != CS_OK) { goto error_no_destroy; } error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, *handle, (void *)&quorum_inst)); if (error != CS_OK) { goto error_destroy; } error = CS_OK; quorum_inst->finalize = 0; quorum_inst->c = qb_ipcc_connect ("quorum", IPC_REQUEST_SIZE); if (quorum_inst->c == NULL) { error = qb_to_cs_error(-errno); goto error_put_destroy; } switch (model) { case QUORUM_MODEL_V0: quorum_gettype_req.size = sizeof (quorum_gettype_req); quorum_gettype_req.id = MESSAGE_REQ_QUORUM_GETTYPE; iov.iov_base = (char *)&quorum_gettype_req; iov.iov_len = sizeof (quorum_gettype_req); error = qb_to_cs_error(qb_ipcc_sendv_recv ( quorum_inst->c, &iov, 1, &res_lib_quorum_gettype, sizeof(res_lib_quorum_gettype), -1)); if (error != CS_OK) { goto error_put_destroy; } error = res_lib_quorum_gettype.header.error; local_quorum_type = res_lib_quorum_gettype.quorum_type; break; case QUORUM_MODEL_V1: quorum_model_gettype_req.header.size = sizeof (quorum_model_gettype_req); quorum_model_gettype_req.header.id = MESSAGE_REQ_QUORUM_MODEL_GETTYPE; quorum_model_gettype_req.model = model; iov.iov_base = (char *)&quorum_model_gettype_req; iov.iov_len = sizeof (quorum_model_gettype_req); error = qb_to_cs_error(qb_ipcc_sendv_recv ( quorum_inst->c, &iov, 1, &res_lib_quorum_model_gettype, sizeof(res_lib_quorum_model_gettype), -1)); if (error != CS_OK) { goto error_put_destroy; } error = res_lib_quorum_model_gettype.header.error; local_quorum_type = res_lib_quorum_model_gettype.quorum_type; break; } if (quorum_type != NULL) { *quorum_type = local_quorum_type; } if (model_data != NULL) { switch (model) { case QUORUM_MODEL_V0: memcpy(&quorum_inst->model_v0_data, model_data, sizeof(quorum_model_v0_data_t)); break; case QUORUM_MODEL_V1: memcpy(&quorum_inst->model_v1_data, model_data, sizeof(quorum_model_v1_data_t)); break; } } quorum_inst->model_data.model = model; quorum_inst->context = context; (void)hdb_handle_put (&quorum_handle_t_db, *handle); return (error); error_put_destroy: (void)hdb_handle_put (&quorum_handle_t_db, *handle); error_destroy: (void)hdb_handle_destroy (&quorum_handle_t_db, *handle); error_no_destroy: return (error); } static void quorum_inst_free (void *inst) { struct quorum_inst *quorum_inst = (struct quorum_inst *)inst; qb_ipcc_disconnect(quorum_inst->c); } cs_error_t quorum_finalize ( quorum_handle_t handle) { struct quorum_inst *quorum_inst; cs_error_t error; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } /* * Another thread has already started finalizing */ if (quorum_inst->finalize) { (void)hdb_handle_put (&quorum_handle_t_db, handle); return (CS_ERR_BAD_HANDLE); } quorum_inst->finalize = 1; (void)hdb_handle_destroy (&quorum_handle_t_db, handle); (void)hdb_handle_put (&quorum_handle_t_db, handle); return (CS_OK); } cs_error_t quorum_getquorate ( quorum_handle_t handle, int *quorate) { cs_error_t error; struct quorum_inst *quorum_inst; struct iovec iov; struct qb_ipc_request_header req; struct res_lib_quorum_getquorate res_lib_quorum_getquorate; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } req.size = sizeof (req); req.id = MESSAGE_REQ_QUORUM_GETQUORATE; iov.iov_base = (char *)&req; iov.iov_len = sizeof (req); error = qb_to_cs_error(qb_ipcc_sendv_recv ( quorum_inst->c, &iov, 1, &res_lib_quorum_getquorate, sizeof (struct res_lib_quorum_getquorate), CS_IPC_TIMEOUT_MS)); if (error != CS_OK) { goto error_exit; } error = res_lib_quorum_getquorate.header.error; *quorate = res_lib_quorum_getquorate.quorate; error_exit: (void)hdb_handle_put (&quorum_handle_t_db, handle); return (error); } cs_error_t quorum_fd_get ( quorum_handle_t handle, int *fd) { cs_error_t error; struct quorum_inst *quorum_inst; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } error = qb_to_cs_error(qb_ipcc_fd_get (quorum_inst->c, fd)); (void)hdb_handle_put (&quorum_handle_t_db, handle); return (error); } cs_error_t quorum_context_get ( quorum_handle_t handle, const void **context) { cs_error_t error; struct quorum_inst *quorum_inst; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } *context = quorum_inst->context; (void)hdb_handle_put (&quorum_handle_t_db, handle); return (CS_OK); } cs_error_t quorum_context_set ( quorum_handle_t handle, const void *context) { cs_error_t error; struct quorum_inst *quorum_inst; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } quorum_inst->context = context; (void)hdb_handle_put (&quorum_handle_t_db, handle); return (CS_OK); } cs_error_t quorum_trackstart ( quorum_handle_t handle, unsigned int flags ) { cs_error_t error; struct quorum_inst *quorum_inst; struct iovec iov; struct req_lib_quorum_trackstart req_lib_quorum_trackstart; struct qb_ipc_response_header res; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } req_lib_quorum_trackstart.header.size = sizeof (struct req_lib_quorum_trackstart); req_lib_quorum_trackstart.header.id = MESSAGE_REQ_QUORUM_TRACKSTART; req_lib_quorum_trackstart.track_flags = flags; iov.iov_base = (char *)&req_lib_quorum_trackstart; iov.iov_len = sizeof (struct req_lib_quorum_trackstart); error = qb_to_cs_error(qb_ipcc_sendv_recv ( quorum_inst->c, &iov, 1, &res, sizeof (res), CS_IPC_TIMEOUT_MS)); if (error != CS_OK) { goto error_exit; } error = res.error; error_exit: (void)hdb_handle_put (&quorum_handle_t_db, handle); return (error); } cs_error_t quorum_trackstop ( quorum_handle_t handle) { cs_error_t error; struct quorum_inst *quorum_inst; struct iovec iov; struct qb_ipc_request_header req; struct qb_ipc_response_header res; error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } req.size = sizeof (req); req.id = MESSAGE_REQ_QUORUM_TRACKSTOP; iov.iov_base = (char *)&req; iov.iov_len = sizeof (req); error = qb_to_cs_error(qb_ipcc_sendv_recv ( quorum_inst->c, &iov, 1, &res, sizeof (res), CS_IPC_TIMEOUT_MS)); if (error != CS_OK) { goto error_exit; } error = res.error; error_exit: (void)hdb_handle_put (&quorum_handle_t_db, handle); return (error); } cs_error_t quorum_dispatch ( quorum_handle_t handle, cs_dispatch_flags_t dispatch_types) { int timeout = -1; cs_error_t error; int cont = 1; /* always continue do loop except when set to 0 */ struct quorum_inst *quorum_inst; struct quorum_inst quorum_inst_copy; struct qb_ipc_response_header *dispatch_data; char dispatch_buf[IPC_DISPATCH_SIZE]; struct res_lib_quorum_notification *res_lib_quorum_notification; struct res_lib_quorum_v1_quorum_notification *res_lib_quorum_v1_quorum_notification; struct res_lib_quorum_v1_nodelist_notification *res_lib_quorum_v1_nodelist_notification; struct quorum_ring_id ring_id; mar_uint32_t *joined_list; mar_uint32_t *left_list; if (dispatch_types != CS_DISPATCH_ONE && dispatch_types != CS_DISPATCH_ALL && dispatch_types != CS_DISPATCH_BLOCKING && dispatch_types != CS_DISPATCH_ONE_NONBLOCKING) { return (CS_ERR_INVALID_PARAM); } error = hdb_error_to_cs(hdb_handle_get (&quorum_handle_t_db, handle, (void *)&quorum_inst)); if (error != CS_OK) { return (error); } /* * Timeout instantly for CS_DISPATCH_ONE_NONBLOCKING or CS_DISPATCH_ALL and * wait indefinately for CS_DISPATCH_ONE or CS_DISPATCH_BLOCKING */ if (dispatch_types == CS_DISPATCH_ALL || dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) { timeout = 0; } dispatch_data = (struct qb_ipc_response_header *)dispatch_buf; do { error = qb_to_cs_error (qb_ipcc_event_recv ( quorum_inst->c, dispatch_buf, IPC_DISPATCH_SIZE, timeout)); if (error == CS_ERR_BAD_HANDLE) { error = CS_OK; goto error_put; } if (error == CS_ERR_TRY_AGAIN) { if (dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) { /* * Don't mask error */ goto error_put; } error = CS_OK; if (dispatch_types == CS_DISPATCH_ALL) { break; /* exit do while cont is 1 loop */ } else { continue; /* next poll */ } } if (error != CS_OK) { goto error_put; } /* * Make copy of callbacks, message data, unlock instance, and call callback * A risk of this dispatch method is that the callback routines may * operate at the same time that quorum_finalize has been called in another thread. */ memcpy (&quorum_inst_copy, quorum_inst, sizeof(quorum_inst_copy)); switch (quorum_inst_copy.model_data.model) { case QUORUM_MODEL_V0: /* * Dispatch incoming message */ switch (dispatch_data->id) { case MESSAGE_RES_QUORUM_NOTIFICATION: if (quorum_inst_copy.model_v0_data.quorum_notify_fn == NULL) { break; } res_lib_quorum_notification = (struct res_lib_quorum_notification *)dispatch_data; quorum_inst_copy.model_v0_data.quorum_notify_fn ( handle, res_lib_quorum_notification->quorate, res_lib_quorum_notification->ring_seq, res_lib_quorum_notification->view_list_entries, res_lib_quorum_notification->view_list); break; default: error = CS_ERR_LIBRARY; goto error_put; break; } /* switch (dispatch_data->id) */ break; /* case QUORUM_MODEL_V0 */ case QUORUM_MODEL_V1: /* * Dispatch incoming message */ switch (dispatch_data->id) { case MESSAGE_RES_QUORUM_V1_QUORUM_NOTIFICATION: if (quorum_inst_copy.model_v1_data.quorum_notify_fn == NULL) { break; } res_lib_quorum_v1_quorum_notification = (struct res_lib_quorum_v1_quorum_notification *)dispatch_data; ring_id.nodeid = res_lib_quorum_v1_quorum_notification->ring_id.nodeid; ring_id.seq = res_lib_quorum_v1_quorum_notification->ring_id.seq; quorum_inst_copy.model_v1_data.quorum_notify_fn ( handle, res_lib_quorum_v1_quorum_notification->quorate, ring_id, res_lib_quorum_v1_quorum_notification->view_list_entries, res_lib_quorum_v1_quorum_notification->view_list); break; case MESSAGE_RES_QUORUM_V1_NODELIST_NOTIFICATION: if (quorum_inst_copy.model_v1_data.nodelist_notify_fn == NULL) { break; } res_lib_quorum_v1_nodelist_notification = (struct res_lib_quorum_v1_nodelist_notification *)dispatch_data; ring_id.nodeid = res_lib_quorum_v1_nodelist_notification->ring_id.nodeid; ring_id.seq = res_lib_quorum_v1_nodelist_notification->ring_id.seq; joined_list = res_lib_quorum_v1_nodelist_notification->member_list + res_lib_quorum_v1_nodelist_notification->member_list_entries; left_list = joined_list + res_lib_quorum_v1_nodelist_notification->joined_list_entries; quorum_inst_copy.model_v1_data.nodelist_notify_fn ( handle, ring_id, res_lib_quorum_v1_nodelist_notification->member_list_entries, res_lib_quorum_v1_nodelist_notification->member_list, res_lib_quorum_v1_nodelist_notification->joined_list_entries, joined_list, res_lib_quorum_v1_nodelist_notification->left_list_entries, left_list); break; default: error = CS_ERR_LIBRARY; goto error_put; break; } /* switch (dispatch_data->id) */ break; /* case QUORUM_MODEL_V1 */ } if (quorum_inst->finalize) { /* * If the finalize has been called then get out of the dispatch. */ error = CS_ERR_BAD_HANDLE; goto error_put; } /* * Determine if more messages should be processed */ if (dispatch_types == CS_DISPATCH_ONE || dispatch_types == CS_DISPATCH_ONE_NONBLOCKING) { cont = 0; } } while (cont); error_put: (void)hdb_handle_put (&quorum_handle_t_db, handle); return (error); }