diff options
Diffstat (limited to '')
45 files changed, 10240 insertions, 0 deletions
diff --git a/lrm/Makefile.am b/lrm/Makefile.am new file mode 100644 index 0000000..78a92c4 --- /dev/null +++ b/lrm/Makefile.am @@ -0,0 +1,20 @@ +# Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright (c) 2004 International Business Machines +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +SUBDIRS = lrmd admin test diff --git a/lrm/admin/Makefile.am b/lrm/admin/Makefile.am new file mode 100644 index 0000000..a92cd72 --- /dev/null +++ b/lrm/admin/Makefile.am @@ -0,0 +1,40 @@ +# +# Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright (c) 2004 International Business Machines +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl + +halibdir = $(libdir)/@HB_PKG@ +COMMONLIBS = $(top_builddir)/lib/clplumbing/libplumb.la $(GLIBLIB) +LRM_DIR = lrm +sbin_PROGRAMS = lrmadmin +sbin_SCRIPTS = cibsecret +lrmadmin_SOURCES = lrmadmin.c +lrmadmin_LDFLAGS = $(COMMONLIBS) +lrmadmin_LDADD = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la +lrmadmin_DEPENDENCIES = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la + +if BUILD_HELP +man8_MANS = $(sbin_PROGRAMS:%=%.8) +%.8: % + echo Creating $@ + chmod a+x $< + help2man --output $@ --no-info --section 8 --name "Part of the Linux-HA project" $(top_builddir)/lrm/admin/$< +endif diff --git a/lrm/admin/cibsecret.in b/lrm/admin/cibsecret.in new file mode 100755 index 0000000..5255cdd --- /dev/null +++ b/lrm/admin/cibsecret.in @@ -0,0 +1,350 @@ +#!/bin/sh + +# Copyright (C) 2011 Dejan Muhamedagic <dmuhamedagic@suse.de> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# WARNING: +# +# The CIB secrets interface and implementation is still being +# discussed, it may change + +# +# cibsecret: manage the secrets directory /var/lib/heartbeat/lrm/secrets +# +# secrets are ascii files, holding just one value per file: +# /var/lib/heartbeat/lrm/secrets/<rsc>/<param> +# +# NB: this program depends on utillib.sh +# + +. @OCF_ROOT_DIR@/lib/heartbeat/ocf-shellfuncs + +HA_NOARCHBIN=@datadir@/@PACKAGE_NAME@ + +. $HA_NOARCHBIN/utillib.sh + +LRM_CIBSECRETS=$HA_VARLIB/lrm/secrets + +PROG=`basename $0` +SSH_OPTS="-o StrictHostKeyChecking=no" + +usage() { + cat<<EOF +usage: $PROG [-C] <command> <parameters> + +-C: don't read/write the CIB + +command: set | delete | stash | unstash | get | check | sync + + set <rsc> <param> <value> + get <rsc> <param> + check <rsc> <param> + stash <rsc> <param> (if not -C) + unstash <rsc> <param> (if not -C) + delete <rsc> <param> + sync + +stash/unstash: move the parameter from/to the CIB (if you already + have the parameter set in the CIB). + +set/delete: add/remove a parameter from the local file. + +get: display the parameter from the local file. + +check: verify MD5 hash of the parameter from the local file and the CIB. + +sync: copy $LRM_CIBSECRETS to other nodes. + +Examples: + + $PROG set ipmi_node1 passwd SecreT_PASS + $PROG stash ipmi_node1 passwd + $PROG get ipmi_node1 passwd + $PROG check ipmi_node1 passwd + $PROG sync +EOF + exit $1 +} +fatal() { + echo "ERROR: $*" + exit 1 +} +warn() { + echo "WARNING: $*" +} +info() { + echo "INFO: $*" +} + +check_env() { + which md5sum >/dev/null 2>&1 || + fatal "please install md5sum to run $PROG" + if which pssh >/dev/null 2>&1; then + rsh=pssh_fun + rcp=pscp_fun + elif which pdsh >/dev/null 2>&1; then + rsh=pdsh_fun + rcp=pdcp_fun + elif which ssh >/dev/null 2>&1; then + rsh=ssh_fun + rcp=scp_fun + else + fatal "please install pssh, pdsh, or ssh to run $PROG" + fi + ps -ef | grep '[c]rmd' >/dev/null || + fatal "pacemaker not running? $PROG needs pacemaker" +} + +get_other_nodes() { + crm_node -l | awk '{print $2}' | grep -v `uname -n` +} +check_down_nodes() { + local n down_nodes + down_nodes=`(for n; do echo $n; done) | sort | uniq -u` + if [ -n "$down_nodes" ]; then + if [ `echo $down_nodes | wc -w` = 1 ]; then + warn "node $down_nodes is down" + warn "you'll need to update it using $PROG sync later" + else + warn "nodes `echo $down_nodes` are down" + warn "you'll need to update them using $PROG sync later" + fi + fi +} + +pssh_fun() { + pssh -qi -H "$nodes" -x "$SSH_OPTS" $* +} +pscp_fun() { + pscp -q -H "$nodes" -x "-pr" -x "$SSH_OPTS" $* +} +pdsh_fun() { + local pdsh_nodes=`echo $nodes | tr ' ' ','` + export PDSH_SSH_ARGS_APPEND="$SSH_OPTS" + pdsh -w $pdsh_nodes $* +} +pdcp_fun() { + local pdsh_nodes=`echo $nodes | tr ' ' ','` + export PDSH_SSH_ARGS_APPEND="$SSH_OPTS" + pdcp -pr -w $pdsh_nodes $* +} +ssh_fun() { + local h + for h in $nodes; do + ssh $SSH_OPTS $h $* || return + done +} +scp_fun() { + local h src="$1" dest=$2 + for h in $nodes; do + scp -pr -q $SSH_OPTS $src $h:$dest || return + done +} +# TODO: this procedure should be replaced with csync2 +# provided that csync2 has already been configured +sync_files() { + local crm_nodes=`get_other_nodes` + local nodes=`get_live_nodes $crm_nodes` + check_down_nodes $nodes $crm_nodes + [ "$nodes" = "" ] && { + info "no other nodes live" + return + } + info "syncing $LRM_CIBSECRETS to `echo $nodes` ..." + $rsh rm -rf $LRM_CIBSECRETS && + $rsh mkdir -p `dirname $LRM_CIBSECRETS` && + $rcp $LRM_CIBSECRETS `dirname $LRM_CIBSECRETS` +} +sync_one() { + local f=$1 f_all="$1 $1.sign" + local crm_nodes=`get_other_nodes` + local nodes=`get_live_nodes $crm_nodes` + check_down_nodes $nodes $crm_nodes + [ "$nodes" = "" ] && { + info "no other nodes live" + return + } + info "syncing $f to `echo $nodes` ..." + $rsh mkdir -p `dirname $f` && + if [ -f "$f" ]; then + $rcp "$f_all" `dirname $f` + else + $rsh rm -f $f_all + fi +} + +is_secret() { + # assume that the secret is in the CIB if we cannot talk to + # cib + [ "$NO_CRM" ] || + test "$1" = "$MAGIC" +} +check_cib_rsc() { + local rsc=$1 output + output=`$NO_CRM crm_resource -r $rsc -W >/dev/null 2>&1` || + fatal "resource $rsc doesn't exist: $output" +} +get_cib_param() { + local rsc=$1 param=$2 + check_cib_rsc $rsc + $NO_CRM crm_resource -r $rsc -g $param 2>/dev/null +} +set_cib_param() { + local rsc=$1 param=$2 value=$3 + check_cib_rsc $rsc + $NO_CRM crm_resource -r $rsc -p $param -v "$value" 2>/dev/null +} +remove_cib_param() { + local rsc=$1 param=$2 + check_cib_rsc $rsc + $NO_CRM crm_resource -r $rsc -d $param 2>/dev/null +} + +localfiles() { + local cmd=$1 + local rsc=$2 param=$3 value=$4 + local local_file=$LRM_CIBSECRETS/$rsc/$param + case $cmd in + "get") + cat $local_file 2>/dev/null + true + ;; + "getsum") + cat $local_file.sign 2>/dev/null + true + ;; + "set") + local md5sum + md5sum=`printf $value | md5sum` || + fatal "md5sum failed to produce hash for resource $rsc parameter $param" + md5sum=`echo $md5sum | awk '{print $1}'` + mkdir -p `dirname $local_file` && + echo $value > $local_file && + echo $md5sum > $local_file.sign && + sync_one $local_file + ;; + "remove") + rm -f $local_file + sync_one $local_file + ;; + *) + # not reached, this is local interface + ;; + esac +} +get_local_param() { + local rsc=$1 param=$2 + localfiles get $rsc $param +} +set_local_param() { + local rsc=$1 param=$2 value=$3 + localfiles set $rsc $param $value +} +remove_local_param() { + local rsc=$1 param=$2 + localfiles remove $rsc $param +} + +cibsecret_set() { + local value=$1 + + if [ -z "$NO_CRM" ]; then + [ "$current" -a "$current" != "$MAGIC" -a "$current" != "$value" ] && + fatal "CIB value <$current> different for $rsc parameter $param; please delete it first" + fi + set_local_param $rsc $param $value && + set_cib_param $rsc $param "$MAGIC" +} + +cibsecret_check() { + local md5sum local_md5sum + is_secret "$current" || + fatal "resource $rsc parameter $param not set as secret, nothing to check" + local_md5sum=`localfiles getsum $rsc $param` + [ "$local_md5sum" ] || + fatal "no MD5 hash for resource $rsc parameter $param" + md5sum=`printf "$current_local" | md5sum | awk '{print $1}'` + [ "$md5sum" = "$local_md5sum" ] || + fatal "MD5 hash mismatch for resource $rsc parameter $param" +} + +cibsecret_get() { + cibsecret_check + echo "$current_local" +} + +cibsecret_delete() { + remove_local_param $rsc $param && + remove_cib_param $rsc $param +} + +cibsecret_stash() { + [ "$NO_CRM" ] && + fatal "no access to Pacemaker, stash not supported" + [ "$current" = "" ] && + fatal "nothing to stash for resource $rsc parameter $param" + is_secret "$current" && + fatal "resource $rsc parameter $param already set as secret, nothing to stash" + cibsecret_set "$current" +} + +cibsecret_unstash() { + [ "$NO_CRM" ] && + fatal "no access to Pacemaker, unstash not supported" + [ "$current_local" = "" ] && + fatal "nothing to unstash for resource $rsc parameter $param" + is_secret "$current" || + warn "resource $rsc parameter $param not set as secret, but we have local value so proceeding anyway" + remove_local_param $rsc $param && + set_cib_param $rsc $param $current_local +} + +cibsecret_sync() { + sync_files +} + +check_env + +MAGIC="lrm://" +umask 0077 + +if [ "$1" = "-C" ]; then + NO_CRM=':' + shift 1 +fi + +cmd=$1 +rsc=$2 +param=$3 +value=$4 + +case "$cmd" in + set) [ $# -ne 4 ] && usage 1;; + get) [ $# -ne 3 ] && usage 1;; + check) [ $# -ne 3 ] && usage 1;; + stash) [ $# -ne 3 ] && usage 1;; + unstash) [ $# -ne 3 ] && usage 1;; + delete) [ $# -ne 3 ] && usage 1;; + sync) [ $# -ne 1 ] && usage 1;; + *) usage 1; +esac + +# we'll need these two often +current=`get_cib_param $rsc $param` +current_local=`get_local_param $rsc $param` + +cibsecret_$cmd $value diff --git a/lrm/admin/lrmadmin.c b/lrm/admin/lrmadmin.c new file mode 100644 index 0000000..27f37bf --- /dev/null +++ b/lrm/admin/lrmadmin.c @@ -0,0 +1,1129 @@ +/* File: lrmadmin.c + * Description: A adminstration tool for Local Resource Manager + * + * Author: Sun Jiang Dong <sunjd@cn.ibm.com> + * Copyright (c) 2004 International Business Machines + * + * Todo: security verification + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#include <lha_internal.h> + +#include <stdio.h> +#include <sys/types.h> +#include <unistd.h> +#include <stdlib.h> +#include <time.h> +#ifndef __USE_GNU +#define __USE_GNU +/* For strnlen protype */ +#include <string.h> +#undef __USE_GNU +#else +#include <string.h> +#endif +#include <errno.h> +#ifdef HAVE_GETOPT_H +#include <getopt.h> +#endif /* HAVE_GETOPT_H */ +#include <clplumbing/cl_log.h> +#include <lrm/lrm_api.h> +#include <lrm/lrm_msg.h> +#include <lrm/raexec.h> +#include <clplumbing/lsb_exitcodes.h> +#include <clplumbing/GSource.h> +#include <clplumbing/Gmain_timeout.h> + +static const char *optstring = "A:D:X:dE:F:dg:p:M:O:P:c:S:LI:CT:n:hv"; + +#ifdef HAVE_GETOPT_H +static struct option long_options[] = { + {"daemon", 0, NULL, 'd'}, + {"executera", 1, NULL, 'E'}, + {"flush", 1, NULL, 'F'}, + {"state", 1, NULL, 'S'}, + {"listall", 0, NULL, 'L'}, + {"information", 1, NULL, 'I'}, + {"add", 1, NULL, 'A'}, + {"delete", 1, NULL, 'D'}, + {"fail", 1, NULL, 'X'}, + {"raclass_supported", 1, NULL, 'C'}, + {"ratype_supported", 1, NULL, 'T'}, + {"all_type_metadata", 1, NULL, 'O'}, + {"metadata", 1, NULL, 'M'}, + {"provider", 1, NULL, 'P'}, + {"set_lrmd_param", 1, NULL, 'p'}, + {"get_lrmd_param", 1, NULL, 'g'}, + {"help", 0, NULL, 'h'}, + {"version", 0, NULL, 'v'}, + {NULL, 0, NULL, 0} +}; +#endif /* HAVE_GETOPT_H */ + +static GMainLoop *mainloop; +static const char *lrmadmin_name = "lrmadmin"; +static const char *fake_name; +/* 20 is the length limit for a argv[x] */ +static const int ARGVI_MAX_LEN = 48; + +typedef enum { + ERROR_OPTION = -1, + NULL_OP, + DAEMON_OP, + EXECUTE_RA, + FLUSH, + RSC_STATE, + LIST_ALLRSC, + INF_RSC, + SET_PARAM, + GET_PARAM, + ADD_RSC, + DEL_RSC, + FAIL_RSC, + RACLASS_SUPPORTED, + RATYPE_SUPPORTED, + RA_METADATA, + RA_PROVIDER, + ALL_RA_METADATA, + HELP +} lrmadmin_cmd_t; + +#define nullcheck(p) ((p) ? (p) : "<null>") +static const char * status_msg[6] = { + "pending", /* LRM_OP_PENDING */ + "succeed", /* LRM_OP_DONE */ + "cancelled", /* LRM_OP_CANCELLED */ + "timeout", /* LRM_OP_TIMEOUT */ + "not Supported", /* LRM_OP_NOTSUPPORTED */ + "failed due to an error" /* LRM_OP_ERROR */ +}; + +static const char * rc_msg[] = { + "unknown error", + "no ra", + "ok", + "unknown error", + "invalid parameter", + "unimplement feature", + "insufficient priority", + "not installed", + "not configured", + "not running", + "running master", + "failed master", + "invalid rc", + /* For status command only */ + "daemon dead1", + "daemon dead2", + "daemon stopped", + "status unknow" +}; + + +static gboolean QUIT_GETOPT = FALSE; +static lrmadmin_cmd_t lrmadmin_cmd = NULL_OP; +static gboolean ASYN_OPS = FALSE; +static int call_id = 0; +static int TIMEOUT = -1; /* the unit is ms */ + +static const char *simple_help_screen = +"lrmadmin -d,--daemon\n" +" -A,--add <rscid> <raclass> <ratype> <provider|NULL> [<rsc_params_list>]\n" +" -D,--delete <rscid>\n" +" -F,--flush <rscid>\n" +" -X,--fail <rscid> [<fail_rc> [<fail_reason>]]\n" +" -E,--execute <rscid> <operator> <timeout> <interval> <target_rc|EVERYTIME|CHANGED> [<operator_parameters_list>]\n" +" -S,--state <rscid> [-n <fake_name>]\n" +" -L,--listall\n" +" -I,--information <rsc_id>\n" +" -C,--raclass_supported\n" +" -T,--ratype_supported <raclass>\n" +" -O,--all metadata of this class <raclass>\n" +" -M,--metadata <raclass> <ratype> <provider|NULL>\n" +" -P,--provider <raclass> <ratype>\n" +" -p,--set_lrmd_param <name> <value>\n" +" -g,--get_lrmd_param <name>\n" +" -v,--version\n" +" -h,--help\n"; + +#define OPTION_OBSCURE_CHECK \ + if ( lrmadmin_cmd != NULL_OP ) { \ + cl_log(LOG_ERR,"Obscure options."); \ + return -1; \ + } + +/* the begin of the internal used function list */ +static int resource_operation(ll_lrm_t * lrmd, char *rsc_id, + int argc, int optind, char * argv[]); +static int add_resource(ll_lrm_t * lrmd, char *rsc_id, + int argc, int optind, char * argv[]); +static int fail_resource(ll_lrm_t * lrmd, char *rsc_id, int optc, char *opts[]); +static int get_param(ll_lrm_t * lrmd, int argc, int optind, char * argv[]); +static int set_param(ll_lrm_t * lrmd, int argc, int optind, char * argv[]); +static int transfer_cmd_params(int amount, int start, char * argv[], + const char * class, GHashTable ** params_ht); +static void g_print_stringitem_and_free(gpointer data, gpointer user_data); +static void g_print_rainfo_item_and_free(gpointer data, gpointer user_data); +static void g_print_ops(gpointer data, gpointer user_data); +static void g_get_rsc_description(gpointer data, gpointer user_data); +static void g_print_meta(gpointer key, gpointer value, gpointer user_data); + +static void print_rsc_inf(lrm_rsc_t * lrmrsc); +static char * params_hashtable_to_str(const char * class, GHashTable * ht); +static void free_stritem_of_hashtable(gpointer key, gpointer value, + gpointer user_data); +static void ocf_params_hash_to_str(gpointer key, gpointer value, + gpointer user_data); +static void normal_params_hash_to_str(gpointer key, gpointer value, + gpointer user_data); +static lrm_rsc_t * get_lrm_rsc(ll_lrm_t * lrmd, char * rscid); + +static int ra_metadata(ll_lrm_t * lrmd, int argc, int optind, char * argv[]); +static int ra_provider(ll_lrm_t * lrmd, int argc, int optind, char * argv[]); +static gboolean lrmd_output_dispatch(IPC_Channel* notused, gpointer user_data); +static gboolean lrm_op_timeout(gpointer data); + +/* the end of the internal used function list */ + +static void lrm_op_done_callback(lrm_op_t* op); + +static int ret_value; +int main(int argc, char **argv) +{ + int option_char; + char rscid_arg_tmp[RID_LEN]; + ll_lrm_t* lrmd; + lrm_rsc_t * lrm_rsc; + GList *raclass_list = NULL, + *ratype_list = NULL, + *rscid_list; + GHashTable *all_meta = NULL; + char raclass[20]; + const char * login_name = lrmadmin_name; + + /* Prevent getopt_long to print error message on stderr itself */ + /*opterr = 0; */ + + if (argc == 1) { + printf("%s",simple_help_screen); + return 0; + } + + cl_log_set_entity(lrmadmin_name); + cl_log_enable_stderr(TRUE); + cl_log_set_facility(LOG_USER); + + memset(rscid_arg_tmp, '\0', RID_LEN); + memset(raclass, '\0', 20); + do { +#ifdef HAVE_GETOPT_H + option_char = getopt_long (argc, argv, optstring, + long_options, NULL); +#else + option_char = getopt (argc, argv, optstring); +#endif + + if (option_char == -1) { + break; + } + + switch (option_char) { + case 'd': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = DAEMON_OP; + QUIT_GETOPT = TRUE; + break; + + case 'A': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = ADD_RSC; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'D': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = DEL_RSC; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'X': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = FAIL_RSC; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'C': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = RACLASS_SUPPORTED; + break; + + case 'T': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = RATYPE_SUPPORTED; + if (optarg) { + strncpy(raclass, optarg, 19); + } + break; + + case 'O': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = ALL_RA_METADATA; + if (optarg) { + strncpy(raclass, optarg, 19); + } + break; + + case 'F': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = FLUSH; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'E': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = EXECUTE_RA; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'M': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = RA_METADATA; + break; + + case 'P': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = RA_PROVIDER; + break; + + case 'S': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = RSC_STATE; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'L': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = LIST_ALLRSC; + break; + + case 'I': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = INF_RSC; + strncpy(rscid_arg_tmp, optarg, RID_LEN-1); + break; + + case 'p': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = SET_PARAM; + break; + + case 'g': + OPTION_OBSCURE_CHECK + lrmadmin_cmd = GET_PARAM; + break; + + case 'n': + if (optarg) { + fake_name = optarg; + } + break; + + case 'v': + printf("%s\n",GLUE_VERSION); + return 0; + case 'h': + OPTION_OBSCURE_CHECK + printf("%s",simple_help_screen); + return 0; + + case '?': + /* cl_log(LOG_ERR,"There is a unrecognized + option %s", optarg); + */ + printf("%s", simple_help_screen); + return -1; + + default: + cl_log(LOG_ERR,"getopt returned character" + " code %c.", option_char); + return -1; + } + } while (!QUIT_GETOPT); + + lrmd = ll_lrm_new("lrm"); + + if (NULL == lrmd) { + cl_log(LOG_ERR,"ll_lrm_new returned NULL."); + return -2; + } + + lrmd->lrm_ops->set_lrm_callback(lrmd, lrm_op_done_callback); + + if (fake_name != NULL) { + login_name = fake_name; + } + if (lrmd->lrm_ops->signon(lrmd, login_name) != 1) { /* != HA_OK */ + printf("lrmd is not running.\n"); + if (lrmadmin_cmd == DAEMON_OP) { + return LSB_STATUS_STOPPED; + } else { + cl_log(LOG_WARNING,"Can't connect to lrmd!"); + return -2; + } + } + + if (lrmadmin_cmd == DAEMON_OP) { + printf("lrmd is stopped.\n"); + lrmd->lrm_ops->signoff(lrmd); + return 0; + } + + switch (lrmadmin_cmd) { + case EXECUTE_RA: + call_id = resource_operation(lrmd, rscid_arg_tmp, argc, optind, argv); + if (call_id < 0) { + if ( call_id == -2 ) { + cl_log(LOG_ERR, "Failed to operate " + "resource %s due to parameter error." + , argv[optind]); + ret_value = -3; + } + if ( call_id == -1 ) { + cl_log(LOG_WARNING, "Failed! No such " + "resource %s.", argv[optind]); + ret_value = -2; + } + else { + cl_log(LOG_ERR, "Failed to operate " + "resource %s due to unknown error." + , argv[optind]); + ret_value = -3; + } + ASYN_OPS = FALSE; + } else { + /* Return value: HA_OK = 1 Or HA_FAIL = 0 */ + if ( call_id == 0 ) { + cl_log(LOG_ERR, "Resource operation " + "failed." ); + ret_value = -3; + ASYN_OPS = FALSE; + } else { + ASYN_OPS = TRUE; + } + } + break; + + case RA_METADATA: + ra_metadata(lrmd, argc, optind, argv); + ASYN_OPS = FALSE; + break; + case RA_PROVIDER: + ra_provider(lrmd, argc, optind, argv); + ASYN_OPS = FALSE; + break; + + case SET_PARAM: + set_param(lrmd, argc, optind, argv); + ASYN_OPS = FALSE; + break; + + case GET_PARAM: + get_param(lrmd, argc, optind, argv); + ASYN_OPS = FALSE; + break; + + case ADD_RSC: + if (add_resource(lrmd, rscid_arg_tmp, argc, optind, argv) == 0) { + printf("Succeeded in adding this resource.\n"); + } else { + printf("Failed to add this resource.\n"); + ret_value = -3; + } + ASYN_OPS = FALSE; + break; + + case DEL_RSC: + /* Return value: HA_OK = 1 Or HA_FAIL = 0 */ + if (lrmd->lrm_ops->delete_rsc(lrmd, rscid_arg_tmp)==1) { + printf("Succeeded in deleting this resource.\n"); + } else { + printf("Failed to delete this resource.\n"); + ret_value = -3; + } + ASYN_OPS = FALSE; + break; + + case FAIL_RSC: + /* Return value: HA_OK = 1 Or HA_FAIL = 0 */ + if (fail_resource(lrmd, rscid_arg_tmp, + argc-optind, argv+optind) == 1) + { + printf("Succeeded in failing the resource.\n"); + } else { + printf("Failed to fail the resource.\n"); + ret_value = -3; + } + ASYN_OPS = FALSE; + break; + + case FLUSH: + lrm_rsc = get_lrm_rsc(lrmd, rscid_arg_tmp); + if (!(lrm_rsc)) { + ret_value = -3; + } else { + /* Return value: HA_OK = 1 Or HA_FAIL = 0 */ + if (lrm_rsc->ops->flush_ops(lrm_rsc) == 1 ) { + printf("Succeeded in flushing.\n"); + } else { + printf("Failed to flush.\n"); + ret_value = -3; + } + lrm_free_rsc(lrm_rsc); + } + + ASYN_OPS = FALSE; + break; + + case RACLASS_SUPPORTED: + raclass_list = lrmd->lrm_ops-> + get_rsc_class_supported(lrmd); + printf("There are %d RA classes supported:\n", + g_list_length(raclass_list)); + if (raclass_list) { + g_list_foreach(raclass_list, g_print_stringitem_and_free, + NULL); + g_list_free(raclass_list); + ret_value = LSB_EXIT_OK; + } else { + printf("No RA classes found!\n"); + ret_value = -3; + } + + ASYN_OPS = FALSE; + break; + + case RATYPE_SUPPORTED: + ratype_list = lrmd->lrm_ops-> + get_rsc_type_supported(lrmd, raclass); + printf("There are %d RAs:\n", g_list_length(ratype_list)); + if (ratype_list) { + g_list_foreach(ratype_list, g_print_rainfo_item_and_free, + NULL); + g_list_free(ratype_list); + } + + ASYN_OPS = FALSE; + break; + case ALL_RA_METADATA: + all_meta = lrmd->lrm_ops->get_all_type_metadata(lrmd, raclass); + if (all_meta) { + g_hash_table_foreach(all_meta, g_print_meta, NULL); + g_hash_table_destroy(all_meta); + } + ASYN_OPS = FALSE; + break; + case LIST_ALLRSC: + rscid_list = lrmd->lrm_ops->get_all_rscs(lrmd); + if (rscid_list) { + g_list_foreach(rscid_list, g_get_rsc_description + , lrmd); + g_list_free(rscid_list); + } else + printf("Currently no resources are managed by " + "LRM.\n"); + + ASYN_OPS = FALSE; + break; + + case INF_RSC: + lrm_rsc = get_lrm_rsc(lrmd, rscid_arg_tmp); + if (!(lrm_rsc)) { + ret_value = -3; + } else { + print_rsc_inf(lrm_rsc); + lrm_free_rsc(lrm_rsc); + } + + ASYN_OPS = FALSE; + break; + + case RSC_STATE: + lrm_rsc = get_lrm_rsc(lrmd, rscid_arg_tmp); + if (!(lrm_rsc)) { + ret_value = -3; + } else { + state_flag_t cur_state = LRM_RSC_IDLE; + GList * ops_queue; + ops_queue = lrm_rsc->ops->get_cur_state(lrm_rsc, + &cur_state); + printf("resource state:%s\n", + cur_state==LRM_RSC_IDLE? + "LRM_RSC_IDLE":"LRM_RSC_BUSY"); + + printf("The resource %d operations' " + "information:\n" + , g_list_length(ops_queue)); + if (ops_queue) { + g_list_foreach(ops_queue, + g_print_ops, + NULL); + lrm_free_op_list(ops_queue); + } + lrm_free_rsc(lrm_rsc); + } + + ASYN_OPS = FALSE; + break; + + + default: + fprintf(stderr, "Option %c is not supported yet.\n", + option_char); + ret_value = -1; + ASYN_OPS = FALSE; + break; + } + + if (ASYN_OPS) { + G_main_add_IPC_Channel(G_PRIORITY_LOW, lrmd->lrm_ops->ipcchan(lrmd), + FALSE, lrmd_output_dispatch, lrmd, NULL); + if (TIMEOUT > 0) { + Gmain_timeout_add(TIMEOUT, lrm_op_timeout, &ret_value); + } + + mainloop = g_main_new(FALSE); + printf( "Waiting for lrmd to callback...\n"); + g_main_run(mainloop); + } + + lrmd->lrm_ops->signoff(lrmd); + return ret_value; +} + +static gboolean +lrm_op_timeout(gpointer data) +{ + int * idata = data; + + printf("ERROR: This operation has timed out - no result from lrmd.\n"); + + *idata = -5; + g_main_quit(mainloop); + return FALSE; +} + +static gboolean +lrmd_output_dispatch(IPC_Channel* notused, gpointer user_data) +{ + ll_lrm_t *lrm = (ll_lrm_t*)user_data; + lrm->lrm_ops->rcvmsg(lrm, FALSE); + + g_main_quit(mainloop); + return TRUE; +} + +static void +lrm_op_done_callback(lrm_op_t* op) +{ + if (!op) { + cl_log(LOG_ERR, "In callback function, op is NULL pointer."); + ret_value = -3; + return; + } + + printf("----------------operation--------------\n"); + printf("type:%s\n", op->op_type); + if ( (0 == STRNCMP_CONST(op->op_type, "status") + || 0 == STRNCMP_CONST(op->op_type, "monitor")) && (op->rc == 7) ) { + printf("operation status:%s\n", status_msg[LRM_OP_DONE-LRM_OP_PENDING]); + printf("op_status: %d\n", LRM_OP_DONE); + } else { + printf("operation status:%s\n", status_msg[(op->op_status + - LRM_OP_PENDING) % DIMOF(status_msg)]); + printf("op_status: %d\n", op->op_status); + } + printf("return code: %d\n", op->rc); + printf("output data: \n%s\n", (op->output ? op->output : "[null]")); + printf("---------------------------------------\n\n"); + ret_value = op->rc; +} + +static int +resource_operation(ll_lrm_t * lrmd, char *rsc_id, int argc, int optind, char * argv[]) +{ + GHashTable * params_ht = NULL; + lrm_op_t op = lrm_zero_op; + lrm_rsc_t * lrm_rsc; + int call_id; + + if ((argc - optind) < 3) { + cl_log(LOG_ERR,"Not enough parameters."); + return -2; + } + + lrm_rsc = lrmd->lrm_ops->get_rsc(lrmd, rsc_id); + if (!lrm_rsc) { + return -1; + } + + op.op_type = argv[optind]; + op.timeout = atoi(argv[optind+1]); + + /* When op.timeout!=0, plus additional 1s. Or lrmadmin may time out before + the normal operation result returned from lrmd. This may be redudant, + but harmless. */ + if (0 < op.timeout ) { + TIMEOUT = op.timeout + 1000; + } + op.interval = atoi(argv[optind+2]); + op.user_data = NULL; + op.user_data_len = 0; + if (0 == strcmp(argv[optind+3], "EVERYTIME")) { + op.target_rc = EVERYTIME; + } + else + if (0 == strcmp(argv[optind+3], "CHANGED")) { + op.target_rc = CHANGED; + } + else { + op.target_rc = atoi(argv[optind+3]); + } + + if ((argc - optind) > 3) { + if (0 > transfer_cmd_params(argc, optind+4, argv, + lrm_rsc->class, ¶ms_ht) ) { + return -2; + } + } + op.params = params_ht; + + call_id = lrm_rsc->ops->perform_op(lrm_rsc, &op); + lrm_free_rsc(lrm_rsc); + if (params_ht) { + g_hash_table_foreach(params_ht, free_stritem_of_hashtable, NULL); + g_hash_table_destroy(params_ht); + } + return call_id; +} +static int +ra_metadata(ll_lrm_t * lrmd, int argc, int optind, char * argv[]) +{ + const char * class = argv[optind-1]; + const char * type = argv[optind]; + const char * provider = argv[optind+1]; + char* metadata; + + if(argc < 5) { + cl_log(LOG_ERR,"Not enough parameters."); + return -2; + } + + if (0 == strncmp(provider,"NULL",strlen("NULL"))) { + provider=NULL; + } + + metadata = lrmd->lrm_ops->get_rsc_type_metadata(lrmd, class, type, provider); + if (NULL!=metadata) { + printf ("%s\n", metadata); + g_free (metadata); + } + return 0; +} + +static int +ra_provider(ll_lrm_t * lrmd, int argc, int optind, char * argv[]) +{ + const char * class = argv[optind-1]; + const char * type = argv[optind]; + GList* providers = NULL; + GList* provider = NULL; + + if(argc < 4) { + cl_log(LOG_ERR,"Not enough parameters."); + return -2; + } + + providers = lrmd->lrm_ops->get_rsc_provider_supported(lrmd,class,type); + + while (NULL != (provider = g_list_first(providers))) { + printf("%s\n",(char*)provider->data); + g_free(provider->data); + providers = g_list_remove(providers, provider->data); + } + g_list_free(providers); + return 0; +} + +static int +add_resource(ll_lrm_t * lrmd, char *rsc_id, int argc, int optind, char * argv[]) +{ + const char * class = argv[optind]; + const char * type = argv[optind+1]; + const char * provider = argv[optind+2]; + GHashTable * params_ht = NULL; + int tmp_ret; + + if ((argc - optind) < 3) { + cl_log(LOG_ERR,"Not enough parameters."); + return -2; + } + + if (0 == strncmp(provider, "NULL", strlen("NULL"))) { + provider=NULL; + } + + /* delete Hashtable */ + if ((argc - optind) > 3) { + if ( 0 > transfer_cmd_params(argc, optind+3, argv, class, + ¶ms_ht) ) { + return -1; + } + } + + tmp_ret = lrmd->lrm_ops->add_rsc(lrmd, rsc_id, class, + type, provider, params_ht); + + /*delete params_ht*/ + if (params_ht) { + g_hash_table_foreach(params_ht, free_stritem_of_hashtable, NULL); + g_hash_table_destroy(params_ht); + } + + return (tmp_ret ? 0 : -1); /* tmp_ret is HA_OK=1 or HA_FAIL=0 */ +} + +static int +fail_resource(ll_lrm_t * lrmd, char *rsc_id, int optc, char *opts[]) +{ + int fail_rc = 0; + const char * reason = NULL; + + if (optc > 2) { + cl_log(LOG_ERR,"Bad usage."); + return -2; + } + + if (optc >= 1) + fail_rc = atoi(opts[0]); + if (optc == 2) + reason = opts[1]; + + return lrmd->lrm_ops->fail_rsc(lrmd, rsc_id, fail_rc, reason); +} + +static int +get_param(ll_lrm_t * lrmd, int argc, int optind, char * argv[]) +{ + const char *name = argv[optind-1]; + char *value; + + if ((argc - optind) != 0) { + cl_log(LOG_ERR,"Bad usage."); + return -2; + } + value = lrmd->lrm_ops->get_lrmd_param(lrmd, name); + printf("%s: %s\n", name, value); + return 0; +} + +static int +set_param(ll_lrm_t * lrmd, int argc, int optind, char * argv[]) +{ + const char *name = argv[optind-1]; + const char *value = argv[optind]; + + if ((argc - optind) != 1) { + cl_log(LOG_ERR,"Bad usage."); + return -2; + } + return lrmd->lrm_ops->set_lrmd_param(lrmd, name, value); +} + +static int +transfer_cmd_params(int amount, int start, char * argv[], const char * class, +GHashTable ** params_ht) +{ + int i, len_tmp; + char * delimit, * key, * value; + char buffer[21]; + + if (amount < start) { + return -1; + } + + if ( strncmp("ocf", class, strlen("ocf"))==0 + || strncmp("stonith", class, strlen("stonith"))==0) { + *params_ht = g_hash_table_new(g_str_hash, g_str_equal); + + for (i=start; i<amount; i++) { + delimit = strchr(argv[i], '='); + if (!delimit) { + cl_log(LOG_ERR, "Parameter %s is invalid for " + "the OCF standard.", argv[i]); + goto error_return; /* Have to */ + } + + len_tmp = strnlen(delimit+1, MAX_PARAM_LEN) + 1; + value = g_new(gchar, len_tmp); + strncpy(value, delimit+1, len_tmp); + + len_tmp = strnlen(argv[i], MAX_PARAM_LEN) - strnlen(delimit, MAX_PARAM_LEN); + key = g_new(gchar, len_tmp+1); + key[len_tmp] = '\0'; + strncpy(key, argv[i], len_tmp); + + g_hash_table_insert(*params_ht, key, value); + } + } else if ( strncmp("lsb", class, strlen("lsb")) == 0 + || strncmp("heartbeat", class, strlen("heartbeat")) == 0 ) { + + /* Pay attention: for parameter ordring issue */ + *params_ht = g_hash_table_new(g_str_hash, g_str_equal); + + memset(buffer, '0', 21); + for (i=start; i<amount; i++) { + snprintf(buffer, 20, "%d", i-start+1); + g_hash_table_insert( *params_ht, g_strdup(buffer), + g_strdup(argv[i])); + /* printf("index: %d value: %s \n", i-start+1, argv[i]); */ + } + } else { + fprintf(stderr, "Not supported resource agent class.\n"); + return -1; + } + + return 0; + +error_return: + if (*params_ht) { + g_hash_table_foreach(*params_ht, free_stritem_of_hashtable, NULL); + g_hash_table_destroy(*params_ht); + *params_ht = NULL; + } + return -1; +} + +static char * +params_hashtable_to_str(const char * class, GHashTable * ht) +{ + int i,ht_size; + gchar * params_str = NULL; + GString * gstr_tmp; + gchar * tmp_str; + + if (!ht) { + return NULL; + } + + if ( strncmp("ocf", class, strlen("ocf")) == 0 + || strncmp("stonith", class, strlen("stonith")) == 0) { + gstr_tmp = g_string_new(""); + g_hash_table_foreach(ht, ocf_params_hash_to_str, &gstr_tmp); + params_str = g_new(gchar, gstr_tmp->len+1); + strncpy(params_str, gstr_tmp->str, gstr_tmp->len+1); + g_string_free(gstr_tmp, TRUE); + } else if ( strncmp("lsb", class, strlen("lsb")) == 0 + || strncmp("heartbeat", class, strlen("heartbeat")) == 0 ) { + ht_size = g_hash_table_size(ht); + if (ht_size == 0) { + return NULL; + } + tmp_str = g_new(gchar, ht_size*ARGVI_MAX_LEN); + memset(tmp_str, ' ', ht_size*ARGVI_MAX_LEN); + tmp_str[ht_size*ARGVI_MAX_LEN-1] = '\0'; + g_hash_table_foreach(ht, normal_params_hash_to_str, &tmp_str); + gstr_tmp = g_string_new(""); + for (i=0; i< ht_size; i++) { + gstr_tmp = g_string_append(gstr_tmp + , tmp_str + i*ARGVI_MAX_LEN ); + gstr_tmp = g_string_append(gstr_tmp, " "); + } + params_str = g_new(gchar, gstr_tmp->len+1); + strncpy(params_str, gstr_tmp->str, gstr_tmp->len+1); + g_string_free(gstr_tmp, TRUE); + } else { + fprintf(stderr, "Not supported resource agent class.\n"); + } + + return params_str; +} + +static void +g_print_stringitem_and_free(gpointer data, gpointer user_data) +{ + printf("%s\n", (char*)data); + g_free(data); +} + +static void +g_print_rainfo_item_and_free(gpointer data, gpointer user_data) +{ + printf("%s\n", (char *)data); + g_free(data); +} + + +static void +g_print_ops(gpointer data, gpointer user_data) +{ + lrm_op_t* op = (lrm_op_t*)data; + GString * param_gstr; + time_t run_at=0, rcchange_at=0; + + if (NULL == op) { + cl_log(LOG_ERR, "%s:%d: op==NULL" + , __FUNCTION__, __LINE__); + return; + } + + param_gstr = g_string_new(""); + g_hash_table_foreach(op->params, ocf_params_hash_to_str, ¶m_gstr); + + if( op->t_run ) + run_at=(time_t)op->t_run; + if( op->t_rcchange ) + rcchange_at=(time_t)op->t_rcchange; + printf(" operation '%s' [call_id=%d]:\n" + " start_delay=%d, interval=%d, timeout=%d, app_name=%s\n" + " rc=%d (%s), op_status=%d (%s)\n" + , nullcheck(op->op_type), op->call_id + , op->start_delay, op->interval, op->timeout + , nullcheck(op->app_name), op->rc + , rc_msg[(op->rc-EXECRA_EXEC_UNKNOWN_ERROR) % DIMOF(rc_msg)] + , op->op_status + , status_msg[(op->op_status-LRM_OP_PENDING) % DIMOF(status_msg)] + ); + if( op->t_run || op->t_rcchange ) + printf(" run at: %s" + " last rc change at: %s" + " queue time: %lums, exec time: %lums\n" + , op->t_run ? ctime(&run_at) : "N/A\n" + , op->t_rcchange ? ctime(&rcchange_at) : "N/A\n" + , op->queue_time, op->exec_time + ); + printf(" parameters: %s\n", param_gstr->str); + g_string_free(param_gstr, TRUE); +} + +static void +g_get_rsc_description(gpointer data, gpointer user_data) +{ + ll_lrm_t* lrmd = (ll_lrm_t *)user_data; + lrm_rsc_t * lrm_rsc; + char rsc_id_tmp[RID_LEN]; + + if (!(user_data)) { + return; + } + + memset(rsc_id_tmp, '\0', RID_LEN); + strncpy(rsc_id_tmp, data, RID_LEN-1); + + lrm_rsc = lrmd->lrm_ops->get_rsc(lrmd, rsc_id_tmp); + if (lrm_rsc) { + print_rsc_inf(lrm_rsc); + lrm_free_rsc(lrm_rsc); + } else + cl_log(LOG_ERR, "Invalid resource id: %s.", + rsc_id_tmp); + + g_free(data); +} +static void +g_print_meta(gpointer key, gpointer value, gpointer user_data) +{ + printf("%s\n", (const char*)key); + printf("%s\n", (const char*)value); +} +static void +print_rsc_inf(lrm_rsc_t * lrm_rsc) +{ + char rscid_str_tmp[RID_LEN]; + char * tmp = NULL; + + if (!lrm_rsc) { + return; + } + + rscid_str_tmp[RID_LEN-1] = '\0'; + strncpy(rscid_str_tmp, lrm_rsc->id, RID_LEN-1); + printf("\nResource ID:%s\n", rscid_str_tmp); + printf("Resource agent class:%s\n", lrm_rsc->class); + printf("Resource agent type:%s\n", lrm_rsc->type); + printf("Resource agent provider:%s\n" + , lrm_rsc->provider?lrm_rsc->provider:"default"); + + if (lrm_rsc->params) { + tmp = params_hashtable_to_str(lrm_rsc->class, + lrm_rsc->params); + printf("Resource agent parameters:%s\n" + , (tmp == NULL) ? "No parameter" : tmp); + if (tmp != NULL) { + g_free(tmp); + } + } +} + +static void +free_stritem_of_hashtable(gpointer key, gpointer value, gpointer user_data) +{ + /*printf("key=%s value=%s\n", (char *)key, (char *)value);*/ + g_free(key); + g_free(value); +} + +static void +ocf_params_hash_to_str(gpointer key, gpointer value, gpointer user_data) +{ + GString * gstr_tmp = *(GString **)user_data; + gstr_tmp = g_string_append(gstr_tmp, (char*)key); + gstr_tmp = g_string_append(gstr_tmp, "="); + gstr_tmp = g_string_append(gstr_tmp, (char *)value); + gstr_tmp = g_string_append(gstr_tmp, " "); +} + +static void +normal_params_hash_to_str(gpointer key, gpointer value, gpointer user_data) +{ + gint key_int; + + gchar * str_tmp = *(gchar **) user_data; + if (str_tmp == NULL ) { + return; + } + + key_int = atoi((char *)key) - 1; + if( key_int < 0 ) { + return; + } + strncpy(str_tmp + key_int * ARGVI_MAX_LEN, (char*)value, + ARGVI_MAX_LEN - 1); +} + +static lrm_rsc_t * +get_lrm_rsc(ll_lrm_t * lrmd, char * rscid) +{ + char uuid_str_tmp[RID_LEN]; + lrm_rsc_t * lrm_rsc; + lrm_rsc = lrmd->lrm_ops->get_rsc(lrmd, rscid); + if (!(lrm_rsc)) { + uuid_str_tmp[RID_LEN-1] = '\0'; + strncpy(uuid_str_tmp, rscid, RID_LEN-1); + cl_log(LOG_ERR,"Resource %s does not exist.", uuid_str_tmp); + } + return lrm_rsc; +} + diff --git a/lrm/admin/lrmadmin.txt b/lrm/admin/lrmadmin.txt new file mode 100644 index 0000000..739aa70 --- /dev/null +++ b/lrm/admin/lrmadmin.txt @@ -0,0 +1,60 @@ +# LRM Admin Command-line Interface +# It's a draft +NAME + lrmadmin - Local Resource Manager Commander-line Daministrator Tools + +SYNOPSIS +lrmadmin {-d|--daemon} + {-A|--add} <rscid> <rsc_class> <rsc_type> [<rsc_params_list>] + {-D|--delete} <rscid> + {-F|--flush} <rscid> + {-E|--execute} <rscid> <operator> <timeout> [<operator_parameters_list>] + {-M|--monitor} -s <rscid> <operator> <timeout> <interval> + [<operator_parameters_list>] + {-M|--monitor} {-g|-c} <rscid> + {-S|--status} <rscid> + {-L|--listall} + {-I|--information} <rsc_id> + {-R|--rasupported} + {-h|--help} + +Detailed Explanation for Options + +Lrmd daemon options + {-d|--daemon} +# -s The status of lrmd: running or not running +# -r Reset lrmd (?) + +Resource options + {-A|--add} <rscid> <rsc_class> <rsc_type> [<rsc_params_list>] + Add a resource. + + {-D|--delete} <rscid> + Delete a resource + + {-E|--execute} <rscid> <operator> <timeout> [<operator_parameters_list>] + Let resource agent to performance the operation + + {-F|--flush} <rscid> + Clear all pending operation on the resource agnency + + {-M|--monitor} {-s|-g} <rscid> <interval> + -s rscname Set a monitors on the resource agent + -g Get the information about the monitors on the + resource agent + + {-S|--status} <rscid> + Get the status of current resource agent + + {-L|--listall} + List all available resource agent + + {-I|--information} <rsc_id> + List the information about a resource + + {-R|--rasupported} + List the support types of resource agent such as OCF and etc. + +Other options + {-h|--help} + Display the help screen diff --git a/lrm/lrmd/Makefile.am b/lrm/lrmd/Makefile.am new file mode 100644 index 0000000..3680928 --- /dev/null +++ b/lrm/lrmd/Makefile.am @@ -0,0 +1,42 @@ +# +# Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright (c) 2002 International Business Machines +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl \ + -I$(top_builddir)/linux-ha -I$(top_srcdir)/linux-ha \ + -I$(top_builddir) -I$(top_srcdir) + +halibdir = $(libdir)/@HB_PKG@ + +COMMONLIBS = $(top_builddir)/lib/clplumbing/libplumb.la \ + $(GLIBLIB) +# $(top_builddir)/lib/apphb/libapphb.la + +halib_PROGRAMS = lrmd + +lrmd_SOURCES = lrmd.c audit.c cib_secrets.c lrmd_fdecl.h lrmd.h + +lrmd_LDFLAGS = $(top_builddir)/lib/lrm/liblrm.la \ + $(COMMONLIBS) @LIBLTDL@ \ + $(top_builddir)/lib/pils/libpils.la + +noinst_HEADERS = lrmd_fdecl.h lrmd.h + +# make lrmd's owner as hacluster:haclient? diff --git a/lrm/lrmd/audit.c b/lrm/lrmd/audit.c new file mode 100644 index 0000000..ec92dad --- /dev/null +++ b/lrm/lrmd/audit.c @@ -0,0 +1,191 @@ +/* + * Audit lrmd global data structures + * + * Author: Dejan Muhamedagic <dejan@suse.de> + * Copyright (c) 2007 Novell GmbH + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <lha_internal.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <stdarg.h> +#include <string.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <dirent.h> +#include <pwd.h> +#include <time.h> + +#include <glib.h> +#include <pils/plugin.h> +#include <pils/generic.h> +#include <clplumbing/GSource.h> +#include <clplumbing/lsb_exitcodes.h> +#include <clplumbing/cl_signal.h> +#include <clplumbing/proctrack.h> +#include <clplumbing/coredumps.h> +#include <clplumbing/uids.h> +#include <clplumbing/Gmain_timeout.h> +#include <clplumbing/cl_pidfile.h> +#include <ha_msg.h> +#ifdef ENABLE_APPHB +# include <apphb.h> +#endif + +#include <lrm/lrm_api.h> +#include <lrm/lrm_msg.h> +#include <lrm/raexec.h> +#include <lrmd.h> + +#ifdef DOLRMAUDITS + +extern GHashTable* clients; +extern GHashTable* resources; + +#define ptr_bad(level,p,item,text) \ + lrmd_log(level,"LRMAUDIT: 0x%lx unallocated pointer for: %s(%s)", \ + (unsigned long)p,item,text); +#define ptr_null(level,item,text) \ + lrmd_log(level,"LRMAUDIT: pointer null for: %s(%s)", \ + item,text); + +/* NB: this macro contains return */ +#define ret_on_null(p,item,text) do { \ + if( !p ) { \ + ptr_bad(LOG_INFO,p,item,text); \ + return; \ + } \ +} while(0) +#define log_on_null(p,item,text) do { \ + if( !p ) { \ + ptr_null(LOG_INFO,item,text); \ + } \ +} while(0) + +void +lrmd_audit(const char *function, int line) +{ + lrmd_log(LOG_DEBUG, "LRMAUDIT: in %s:%d",function,line); +#ifdef LRMAUDIT_CLIENTS + audit_clients(); +#endif +#ifdef LRMAUDIT_RESOURCES + audit_resources(); +#endif +} + +void +audit_clients() +{ + g_hash_table_foreach(clients, on_client, NULL); +} + +void +audit_resources() +{ + g_hash_table_foreach(resources, on_resource, NULL); +} + +void +audit_ops(GList* rsc_ops, lrmd_rsc_t* rsc, const char *desc) +{ + GList *oplist; + + for( oplist = g_list_first(rsc_ops); + oplist; oplist = g_list_next(oplist) ) + { + on_op(oplist->data, rsc, desc); + } +} + +void +on_client(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_client_t * client = (lrmd_client_t*)value; + + ret_on_null(client,"","client"); + log_on_null(client->app_name,"","app_name"); + log_on_null(client->ch_cmd,client->app_name,"ch_cmd"); + log_on_null(client->ch_cbk,client->app_name,"ch_cbk"); + log_on_null(client->g_src,client->app_name,"g_src"); + log_on_null(client->g_src_cbk,client->app_name,"g_src_cbk"); +} + +void +on_resource(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_rsc_t* rsc = (lrmd_rsc_t*)value; + + ret_on_null(rsc,"","rsc"); + ret_on_null(rsc->id,"","id"); + log_on_null(rsc->type,rsc->id,"type"); + log_on_null(rsc->class,rsc->id,"class"); + log_on_null(rsc->provider,rsc->id,"provider"); + /*log_on_null(rsc->params,rsc->id,"params");*/ + log_on_null(rsc->last_op_table,rsc->id,"last_op_table"); + log_on_null(rsc->last_op_done,rsc->id,"last_op_done"); + audit_ops(rsc->op_list,rsc,"op_list"); + audit_ops(rsc->repeat_op_list,rsc,"repeat_op_list"); +} + +void +on_op(lrmd_op_t *op, lrmd_rsc_t* rsc, const char *desc) +{ + ret_on_null(op,rsc->id,desc); + log_on_null(op->rsc_id,rsc->id,"rsc_id"); + if( strcmp(op->rsc_id,rsc->id) ) { + lrmd_log(LOG_ERR,"LRMAUDIT: rsc %s, op %s " + "op->rsc_id does not match rsc->id", + rsc->id,small_op_info(op)); + } + log_on_null(op->msg,small_op_info(op),"msg"); + if( op->rapop ) { + if( op->rapop->lrmd_op != op ) { + lrmd_log(LOG_ERR, + "LRMAUDIT: rsc %s, op %s: rapop->lrmd_op does not match op", + rsc->id,small_op_info(op)); + } + if( strcmp(op->rapop->rsc_id,op->rsc_id) ) { + lrmd_log(LOG_ERR, + "LRMAUDIT: rsc %s, op %s rapop->rsc_id does not match op->rsc_id", + rsc->id,small_op_info(op)); + } + on_ra_pipe_op(op->rapop,op,"rapop"); + } +} + +void +on_ra_pipe_op(ra_pipe_op_t *rapop, lrmd_op_t *op, const char *desc) +{ + ret_on_null(rapop,small_op_info(op),desc); + log_on_null(rapop->ra_stdout_gsource,small_op_info(op),"ra_stdout_gsource"); + log_on_null(rapop->ra_stderr_gsource,small_op_info(op),"ra_stderr_gsource"); + log_on_null(rapop->rsc_id,small_op_info(op),"rsc_id"); + log_on_null(rapop->op_type,small_op_info(op),"op_type"); + log_on_null(rapop->rsc_class,small_op_info(op),"rsc_class"); + if( strcmp(op->rsc_id,rapop->rsc_id) ) { + lrmd_log(LOG_ERR,"LRMAUDIT: %s: rapop->rsc_id " + "does not match op_rsc->id", + small_op_info(op)); + } +} + +#endif /*DOLRMAUDITS*/ diff --git a/lrm/lrmd/cib_secrets.c b/lrm/lrmd/cib_secrets.c new file mode 100644 index 0000000..612ffdb --- /dev/null +++ b/lrm/lrmd/cib_secrets.c @@ -0,0 +1,205 @@ +/* + * cib_secrets.c + * + * Author: Dejan Muhamedagic <dejan@suse.de> + * Copyright (c) 2011 SUSE, Attachmate + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <lha_internal.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <time.h> + +#include <glib.h> +#include <pils/plugin.h> +#include <pils/generic.h> +#include <clplumbing/GSource.h> +#include <clplumbing/lsb_exitcodes.h> +#include <clplumbing/cl_signal.h> +#include <clplumbing/proctrack.h> +#include <clplumbing/coredumps.h> +#include <clplumbing/uids.h> +#include <clplumbing/Gmain_timeout.h> +#include <clplumbing/cl_pidfile.h> +#include <clplumbing/realtime.h> +#include <clplumbing/md5.h> +#include <ha_msg.h> + +#include <lrm/lrm_api.h> +#include <lrm/lrm_msg.h> + +#include <lrmd.h> + +int replace_secret_params(char *rsc_id, GHashTable* params); +static int is_magic_value(char *p); +static int check_md5_hash(char *hash, char *value); +static void add_secret_params(gpointer key, gpointer value, gpointer user_data); +static char *read_local_file(char *local_file); + +#define MAGIC "lrm://" + +static int +is_magic_value(char *p) +{ + return !strcmp(p, MAGIC); +} + +#define MD5LEN 16 +static int +check_md5_hash(char *hash, char *value) +{ + int i; + char hash2[2*MD5LEN+1]; + unsigned char binary[MD5LEN+1]; + + MD5((unsigned char *)value, strlen(value), binary); + for (i = 0; i < MD5LEN; i++) + sprintf(hash2+2*i, "%02x", binary[i]); + hash2[2*i] = '\0'; + lrmd_debug2(LOG_DEBUG + , "%s:%d: hash: %s, calculated hash: %s" + , __FUNCTION__, __LINE__, hash, hash2); + return !strcmp(hash, hash2); +} + +static char * +read_local_file(char *local_file) +{ + FILE *fp = fopen(local_file, "r"); + char buf[MAX_VALUE_LEN+1]; + char *p; + + if (!fp) { + if (errno != ENOENT) { + cl_perror("%s:%d: cannot open %s" + , __FUNCTION__, __LINE__, local_file); + } + return NULL; + } + if (!fgets(buf, MAX_VALUE_LEN, fp)) { + cl_perror("%s:%d: cannot read %s" + , __FUNCTION__, __LINE__, local_file); + return NULL; + } + /* strip white space */ + for (p = buf+strlen(buf)-1; p >= buf && isspace(*p); p--) + ; + *(p+1) = '\0'; + return g_strdup(buf); +} + +/* + * returns 0 on success or no replacements necessary + * returns -1 if replacement failed for whatever reasone + */ + +int +replace_secret_params(char *rsc_id, GHashTable* params) +{ + char local_file[FILENAME_MAX+1], *start_pname; + char hash_file[FILENAME_MAX+1], *hash; + GList *secret_params = NULL, *l; + char *key, *pvalue, *secret_value; + int rc = 0; + + /* secret_params could be cached with the resource; + * there are also parameters sent with operations + * which cannot be cached + */ + g_hash_table_foreach(params, add_secret_params, &secret_params); + if (!secret_params) /* none found? */ + return 0; + + lrmd_debug(LOG_DEBUG + , "%s:%d: replace secret parameters for resource %s" + , __FUNCTION__, __LINE__, rsc_id); + if (snprintf(local_file, FILENAME_MAX, + LRM_CIBSECRETS "/%s/", rsc_id) > FILENAME_MAX) { + lrmd_log(LOG_ERR + , "%s:%d: filename size exceeded for resource %s" + , __FUNCTION__, __LINE__, rsc_id); + return -1; + } + start_pname = local_file + strlen(local_file); + + for (l = g_list_first(secret_params); l; l = g_list_next(l)) { + key = (char *)(l->data); + pvalue = g_hash_table_lookup(params, key); + if (!pvalue) { /* this cannot really happen */ + lrmd_log(LOG_ERR + , "%s:%d: odd, no parameter %s for rsc %s found now" + , __FUNCTION__, __LINE__, key, rsc_id); + continue; + } + if ((strlen(key) + strlen(local_file)) >= FILENAME_MAX-2) { + lrmd_log(LOG_ERR + , "%s:%d: parameter name %s too big" + , __FUNCTION__, __LINE__, key); + rc = -1; + continue; + } + strcpy(start_pname, key); + secret_value = read_local_file(local_file); + if (!secret_value) { + lrmd_log(LOG_ERR + , "%s:%d: secret for rsc %s parameter %s " + "not found in " LRM_CIBSECRETS + , __FUNCTION__, __LINE__, rsc_id, key); + rc = -1; + continue; + } + strcpy(hash_file, local_file); + if (strlen(hash_file) + 5 > FILENAME_MAX) { + lrmd_log(LOG_ERR + , "%s:%d: cannot build such a long name " + "for the sign file: %s.sign" + , __FUNCTION__, __LINE__, hash_file); + } else { + strncat(hash_file, ".sign", 5); + hash = read_local_file(hash_file); + if (!check_md5_hash(hash, secret_value)) { + lrmd_log(LOG_ERR + , "%s:%d: md5 sum for rsc %s parameter %s " + "does not match" + , __FUNCTION__, __LINE__, rsc_id, key); + g_free(secret_value); + g_free(hash); + rc = -1; + continue; + } + g_free(hash); + } + g_hash_table_replace(params, g_strdup(key), secret_value); + } + g_list_free(secret_params); + return rc; +} + +static void +add_secret_params(gpointer key, gpointer value, gpointer user_data) +{ + GList **lp = (GList **)user_data; + + if (is_magic_value((char *)value)) + *lp = g_list_append(*lp, (char *)key); +} diff --git a/lrm/lrmd/lrmd.c b/lrm/lrmd/lrmd.c new file mode 100644 index 0000000..385096b --- /dev/null +++ b/lrm/lrmd/lrmd.c @@ -0,0 +1,4053 @@ +/* + * Local Resource Manager Daemon + * + * Author: Huang Zhen <zhenhltc@cn.ibm.com> + * Partly contributed by Andrew Beekhof <andrew@beekhof.net> + * Copyright (c) 2004 International Business Machines + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <lha_internal.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <stdarg.h> +#include <string.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <dirent.h> +#include <pwd.h> +#include <time.h> +#include <sched.h> + +#include <glib.h> +#include <pils/plugin.h> +#include <pils/generic.h> +#include <clplumbing/GSource.h> +#include <clplumbing/lsb_exitcodes.h> +#include <clplumbing/cl_signal.h> +#include <clplumbing/proctrack.h> +#include <clplumbing/coredumps.h> +#include <clplumbing/uids.h> +#include <clplumbing/Gmain_timeout.h> +#include <clplumbing/cl_pidfile.h> +#include <clplumbing/realtime.h> +#include <ha_msg.h> +#ifdef ENABLE_APPHB +# include <apphb.h> +#endif +/* #include <hb_api.h> */ + +#include <lrm/lrm_api.h> +#include <lrm/lrm_msg.h> +#include <lrm/raexec.h> + +#include <lrmd.h> +#include <lrmd_fdecl.h> + +static gboolean in_alloc_dump = FALSE; + +ProcTrack_ops ManagedChildTrackOps = { + on_ra_proc_finished, + on_ra_proc_registered, + on_ra_proc_query_name +}; + +/* msg dispatch table */ +typedef int (*msg_handler)(lrmd_client_t* client, struct ha_msg* msg); +struct msg_map +{ + const char *msg_type; + int reply_time; + msg_handler handler; + int min_priv; /* minimum privileges required */ +}; + +/* + * two ways to handle replies: + * REPLY_NOW: pack whatever the handler returned and send it + * NO_MSG: the handler will send the reply itself + */ +#define REPLY_NOW 0 +#define NO_MSG 1 +#define send_msg_now(p) \ + (p->reply_time==REPLY_NOW) + +struct msg_map msg_maps[] = { + {REGISTER, REPLY_NOW, on_msg_register, 0}, + {GETRSCCLASSES, NO_MSG, on_msg_get_rsc_classes, 0}, + {GETRSCTYPES, NO_MSG, on_msg_get_rsc_types, 0}, + {GETPROVIDERS, NO_MSG, on_msg_get_rsc_providers, 0}, + {ADDRSC, REPLY_NOW, on_msg_add_rsc, PRIV_ADMIN}, + {GETRSC, NO_MSG, on_msg_get_rsc, PRIV_ADMIN}, + {GETLASTOP, NO_MSG, on_msg_get_last_op, PRIV_ADMIN}, + {GETALLRCSES, NO_MSG, on_msg_get_all, PRIV_ADMIN}, + {DELRSC, REPLY_NOW, on_msg_del_rsc, PRIV_ADMIN}, + {FAILRSC, REPLY_NOW, on_msg_fail_rsc, PRIV_ADMIN}, + {PERFORMOP, REPLY_NOW, on_msg_perform_op, PRIV_ADMIN}, + {FLUSHOPS, REPLY_NOW, on_msg_flush_all, PRIV_ADMIN}, + {CANCELOP, REPLY_NOW, on_msg_cancel_op, PRIV_ADMIN}, + {GETRSCSTATE, NO_MSG, on_msg_get_state, PRIV_ADMIN}, + {GETRSCMETA, NO_MSG, on_msg_get_metadata, 0}, + {SETLRMDPARAM, REPLY_NOW, on_msg_set_lrmd_param, PRIV_ADMIN}, + {GETLRMDPARAM, NO_MSG, on_msg_get_lrmd_param, 0}, +}; +#define MSG_NR sizeof(msg_maps)/sizeof(struct msg_map) + +GHashTable* clients = NULL; /* a GHashTable indexed by pid */ +GHashTable* resources = NULL; /* a GHashTable indexed by rsc_id */ + +static GMainLoop* mainloop = NULL; +static int call_id = 1; +static const char* lrm_system_name = "lrmd"; +static GHashTable * RAExecFuncs = NULL; +static GList* ra_class_list = NULL; +static gboolean shutdown_in_progress = FALSE; +static unsigned long apphb_interval = 2000; /* Millisecond */ +static gboolean reg_to_apphbd = FALSE; +static int max_child_count = 4; +static int retry_interval = 1000; /* Millisecond */ +static int child_count = 0; +static IPC_Auth * auth = NULL; + +static struct { + int opcount; + int clientcount; + int rsccount; +}lrm_objectstats; + +/* define indexes into logmsg_ctrl_defs */ +#define OP_STAYED_TOO_LONG 0 +static struct logspam logmsg_ctrl_defs[] = { + { "operation stayed too long in the queue", + 10, 60, 120, /* max 10 messages in 60s, then delay for 120s */ + "configuration advice: reduce operation contention " + "either by increasing lrmd max_children or by increasing intervals " + "of monitor operations" + }, +}; + +#define set_fd_opts(fd,opts) do { \ + int flag; \ + if ((flag = fcntl(fd, F_GETFL)) >= 0) { \ + if (fcntl(fd, F_SETFL, flag|opts) < 0) { \ + cl_perror("%s::%d: fcntl", __FUNCTION__ \ + , __LINE__); \ + } \ + } else { \ + cl_perror("%s::%d: fcntl", __FUNCTION__, __LINE__); \ + } \ + } while(0) + +static ra_pipe_op_t * +ra_pipe_op_new(int child_stdout, int child_stderr, lrmd_op_t * lrmd_op) +{ + ra_pipe_op_t * rapop; + lrmd_rsc_t* rsc = NULL; + + if ( NULL == lrmd_op ) { + lrmd_log(LOG_WARNING + , "%s:%d: lrmd_op==NULL, no need to malloc ra_pipe_op" + , __FUNCTION__, __LINE__); + return NULL; + } + rapop = calloc(sizeof(ra_pipe_op_t), 1); + if ( rapop == NULL) { + lrmd_log(LOG_ERR, "%s:%d out of memory" + , __FUNCTION__, __LINE__); + return NULL; + } + rapop->first_line_read = FALSE; + + /* + * No any obviouse proof of lrmd hang in pipe read yet. + * Bug 475 may be a duplicate of bug 499. + * Anyway, via test, it's proved that NOBLOCK read will + * obviously reduce the RA execution time (bug 553). + */ + /* Let the read operation be NONBLOCK */ + set_fd_opts(child_stdout,O_NONBLOCK); + set_fd_opts(child_stderr,O_NONBLOCK); + + /* there's so much code duplication here */ + rapop->ra_stdout_fd = child_stdout; + if (rapop->ra_stdout_fd <= STDERR_FILENO) { + lrmd_log(LOG_ERR, "%s: invalid stdout fd [%d]" + , __FUNCTION__, rapop->ra_stdout_fd); + } + rapop->ra_stdout_gsource = G_main_add_fd(G_PRIORITY_HIGH + , child_stdout, FALSE, handle_pipe_ra_stdout + , rapop, destroy_pipe_ra_stdout); + + rapop->ra_stderr_fd = child_stderr; + if (rapop->ra_stderr_fd <= STDERR_FILENO) { + lrmd_log(LOG_ERR, "%s: invalid stderr fd [%d]" + , __FUNCTION__, rapop->ra_stderr_fd); + } + rapop->ra_stderr_gsource = G_main_add_fd(G_PRIORITY_HIGH + , child_stderr, FALSE, handle_pipe_ra_stderr + , rapop, destroy_pipe_ra_stderr); + + rapop->lrmd_op = lrmd_op; + + rapop->op_type = strdup(ha_msg_value(lrmd_op->msg, F_LRM_OP)); + rapop->rsc_id = strdup(lrmd_op->rsc_id); + rsc = lookup_rsc(lrmd_op->rsc_id); + if (rsc == NULL) { + lrmd_debug(LOG_WARNING + , "%s::%d: the rsc (id=%s) does not exist" + , __FUNCTION__, __LINE__, lrmd_op->rsc_id); + rapop->rsc_class = NULL; + } else { + rapop->rsc_class = strdup(rsc->class); + } + + return rapop; +} + +static void +ra_pipe_op_destroy(ra_pipe_op_t * rapop) +{ + CHECK_ALLOCATED(rapop, "ra_pipe_op", ); + + if ( NULL != rapop->ra_stdout_gsource) { + G_main_del_fd(rapop->ra_stdout_gsource); + rapop->ra_stdout_gsource = NULL; + } + + if ( NULL != rapop->ra_stderr_gsource) { + G_main_del_fd(rapop->ra_stderr_gsource); + rapop->ra_stderr_gsource = NULL; + } + + if (rapop->ra_stdout_fd >= STDERR_FILENO) { + close(rapop->ra_stdout_fd); + rapop->ra_stdout_fd = -1; + }else if (rapop->ra_stdout_fd >= 0) { + lrmd_log(LOG_ERR, "%s: invalid stdout fd %d" + , __FUNCTION__, rapop->ra_stdout_fd); + } + if (rapop->ra_stderr_fd >= STDERR_FILENO) { + close(rapop->ra_stderr_fd); + rapop->ra_stderr_fd = -1; + }else if (rapop->ra_stderr_fd >= 0) { + lrmd_log(LOG_ERR, "%s: invalid stderr fd %d" + , __FUNCTION__, rapop->ra_stderr_fd); + } + rapop->first_line_read = FALSE; + + free(rapop->rsc_id); + free(rapop->op_type); + rapop->op_type = NULL; + free(rapop->rsc_class); + rapop->rsc_class = NULL; + + if (rapop->lrmd_op != NULL) { + rapop->lrmd_op->rapop = NULL; + rapop->lrmd_op = NULL; + } + + free(rapop); +} + +static void +lrmd_op_destroy(lrmd_op_t* op) +{ + CHECK_ALLOCATED(op, "op", ); + --lrm_objectstats.opcount; + + if (op->exec_pid > 1) { + lrmd_log(LOG_CRIT + , "%s: lingering operation process %d, op %s" + , __FUNCTION__, op->exec_pid, small_op_info(op)); + return; + } + lrmd_debug2(LOG_DEBUG, "%s: free the %s with address %p" + ,__FUNCTION__, op_info(op), op); + ha_msg_del(op->msg); + op->msg = NULL; + if( op->rsc_id ) { + free(op->rsc_id); + op->rsc_id = NULL; + } + op->exec_pid = 0; + if ( op->rapop != NULL ) { + op->rapop->lrmd_op = NULL; + op->rapop = NULL; + } + op->first_line_ra_stdout[0] = EOS; + + if( op->repeat_timeout_tag ) { + Gmain_timeout_remove(op->repeat_timeout_tag); + } + free(op); +} + +static lrmd_op_t* +lrmd_op_new(void) +{ + lrmd_op_t* op = (lrmd_op_t*)calloc(sizeof(lrmd_op_t),1); + + if (op == NULL) { + lrmd_log(LOG_ERR, "lrmd_op_new(): out of memory when " + "calloc a lrmd_op_t."); + return NULL; + } + op->rsc_id = NULL; + op->msg = NULL; + op->exec_pid = -1; + op->repeat_timeout_tag = 0; + op->rapop = NULL; + op->first_line_ra_stdout[0] = EOS; + op->t_recv = time_longclock(); + op->t_perform = zero_longclock; + op->t_done = zero_longclock; + op->t_rcchange = zero_longclock; + op->t_lastlogmsg = zero_longclock; + + memset(op->killseq, 0, sizeof(op->killseq)); + ++lrm_objectstats.opcount; + return op; +} + +static lrmd_op_t* +lrmd_op_copy(const lrmd_op_t* op) +{ + lrmd_op_t* ret; + + ret = lrmd_op_new(); + if (NULL == ret) { + return NULL; + } + /* Do a "shallow" copy */ + *ret = *op; + /* + * Some things, like timer ids and child pids are duplicated here + * but can be destroyed in one copy, but kept intact + * in the other, to later be destroyed. + * This isn't a complete disaster, since the timer ids aren't + * pointers, but it's still untidy at the least. + * Be sure and care of this situation when using this function. + */ + /* Do a "deep" copy of the message structure */ + ret->rapop = NULL; + ret->msg = ha_msg_copy(op->msg); + ret->rsc_id = strdup(op->rsc_id); + ret->rapop = NULL; + ret->first_line_ra_stdout[0] = EOS; + ret->repeat_timeout_tag = 0; + ret->exec_pid = -1; + ret->t_recv = op->t_recv; + ret->t_perform = op->t_perform; + ret->t_done = op->t_done; + ret->t_rcchange = op->t_rcchange; + ret->is_copy = TRUE; + ret->is_cancelled = FALSE; + ret->weight = op->weight; + return ret; +} + +static +const char * +op_status_to_str(int op_status) +{ + static char whatwasthat[25]; + switch (op_status) { + case LRM_OP_DONE: + return "LRM_OP_DONE"; + case LRM_OP_CANCELLED: + return "LRM_OP_CANCELLED"; + case LRM_OP_TIMEOUT: + return "LRM_OP_TIMEOUT"; + case LRM_OP_NOTSUPPORTED: + return "LRM_OP_NOTSUPPORTED"; + case -1: + return "N/A (-1)"; + default: + break; + } + snprintf(whatwasthat, sizeof(whatwasthat), "UNDEFINED STATUS: %d?", op_status); + return whatwasthat; +} +static +const char * +op_target_rc_to_str(int target) +{ + static char whatwasthat[25]; + switch (target) { + case EVERYTIME: + return "EVERYTIME"; + case CHANGED: + return "CHANGED"; + default: + break; + } + snprintf(whatwasthat, sizeof(whatwasthat) + ,"UNDEFINED TARGET_RC: %d", target); + return whatwasthat; +} + +/* + * We need a separate function to dump out operations for + * debugging. Then we wouldn't have to have the code for this + * inline. In particular, we could then call this from on_op_done() + * which would shorten and simplify that code - which could use + * the help :-) + */ + + +/* Debug oriented funtions */ +static gboolean debug_level_adjust(int nsig, gpointer user_data); + +static void +lrmd_op_dump(const lrmd_op_t* op, const char * text) +{ + int op_status = -1; + int target_rc = -1; + const char * pidstat; + longclock_t now = time_longclock(); + + CHECK_ALLOCATED(op, "op", ); + if (op->exec_pid < 1 + || ((kill(op->exec_pid, 0) < 0) && ESRCH == errno)) { + pidstat = "not running"; + }else{ + pidstat = "running"; + } + ha_msg_value_int(op->msg, F_LRM_OPSTATUS, &op_status); + ha_msg_value_int(op->msg, F_LRM_TARGETRC, &target_rc); + lrmd_debug(LOG_DEBUG + , "%s: lrmd_op: %s status: %s, target_rc=%s, client pid %d call_id" + ": %d, child pid: %d (%s) %s %s" + , text, op_info(op), op_status_to_str(op_status) + , op_target_rc_to_str(target_rc) + , op->client_id, op->call_id, op->exec_pid, pidstat + , (op->is_copy ? "copy" : "original") + , (op->is_cancelled ? "cancelled" : "")); + lrmd_debug(LOG_DEBUG + , "%s: lrmd_op2: rt_tag: %d, interval: %d, delay: %d" + , text, op->repeat_timeout_tag + , op->interval, op->delay); + lrmd_debug(LOG_DEBUG + , "%s: lrmd_op3: t_recv: %ldms, t_add: %ldms" + ", t_perform: %ldms, t_done: %ldms, t_rcchange: %ldms" + , text, tm2age(op->t_recv), tm2age(op->t_addtolist) + , tm2age(op->t_perform), tm2age(op->t_done), tm2age(op->t_rcchange)); + lrmd_rsc_dump(op->rsc_id, text); +} + + +static void +lrmd_client_destroy(lrmd_client_t* client) +{ + CHECK_ALLOCATED(client, "client", ); + + --lrm_objectstats.clientcount; + /* + * Delete direct references to this client + * and repeating operations it might have scheduled + */ + unregister_client(client); + if (client->app_name) { + free(client->app_name); + client->app_name = NULL; + } + free(client); +} + +static lrmd_client_t* +lrmd_client_new(void) +{ + lrmd_client_t* client; + client = calloc(sizeof(lrmd_client_t), 1); + if (client == NULL) { + lrmd_log(LOG_ERR, "lrmd_client_new(): out of memory when " + "calloc lrmd_client_t."); + return NULL; + } + client->g_src = NULL; + client->g_src_cbk = NULL; + ++lrm_objectstats.clientcount; + return client; +} +static void +lrmd_client_dump(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_client_t * client = (lrmd_client_t*)value; + CHECK_ALLOCATED(client, "client", ); + if(!client) { + return; + } + + lrmd_debug(LOG_DEBUG, "client name: %s, client pid: %d" + ", client uid: %d, gid: %d, last request: %s" + ", last op in: %s, lastop out: %s" + ", last op rc: %s" + , lrm_str(client->app_name) + , client->pid + , client->uid, client->gid + , client->lastrequest + , ctime(&client->lastreqstart) + , ctime(&client->lastreqend) + , ctime(&client->lastrcsent) + ); + if (!client->ch_cmd) { + lrmd_debug(LOG_DEBUG, "NULL client ch_cmd in %s()", __FUNCTION__); + }else{ + lrmd_debug(LOG_DEBUG + , "Command channel status: %d, read queue addr: %p, write queue addr: %p" + , client->ch_cmd->ch_status + , client->ch_cmd->recv_queue + , client->ch_cmd->send_queue ); + + if (client->ch_cmd->recv_queue && client->ch_cmd->send_queue) { + lrmd_debug(LOG_DEBUG, "read Qlen: %ld, write Qlen: %ld" + , (long)client->ch_cmd->recv_queue->current_qlen + , (long)client->ch_cmd->send_queue->current_qlen); + } + } + if (!client->ch_cbk) { + lrmd_debug(LOG_DEBUG, "NULL client ch_cbk in %s()", __FUNCTION__); + }else{ + lrmd_debug(LOG_DEBUG + , "Callback channel status: %d, read Qlen: %ld, write Qlen: %ld" + , client->ch_cbk->ch_status + , (long)client->ch_cbk->recv_queue->current_qlen + , (long)client->ch_cbk->send_queue->current_qlen); + } +} +static void +lrmd_dump_all_clients(void) +{ + static gboolean incall = FALSE; + + if (incall) { + return; + } + + incall = TRUE; + + lrmd_debug(LOG_DEBUG, "%d clients connected to lrmd" + , g_hash_table_size(clients)); + + g_hash_table_foreach(clients, lrmd_client_dump, NULL); + incall = FALSE; +} + +static void +lrmd_rsc_destroy(lrmd_rsc_t* rsc) +{ + LRMAUDIT(); + CHECK_ALLOCATED(rsc, "resource", ); + --lrm_objectstats.rsccount; + if( rsc->op_list || rsc->repeat_op_list ) { + lrmd_log(LOG_ERR, "%s: refusing to remove resource %s" + " which is still holding operations" + , __FUNCTION__, lrm_str(rsc->id)); + return; + } else { + lrmd_debug(LOG_DEBUG, "%s: removing resource %s" + , __FUNCTION__, lrm_str(rsc->id)); + } + g_hash_table_remove(resources, rsc->id); + if (rsc->id) { + free(rsc->id); + rsc->id = NULL; + } + if (rsc->type) { + free(rsc->type); + rsc->type = NULL; + } + if (rsc->class) { + free(rsc->class); + rsc->class = NULL; + } + if (rsc->provider) { + free(rsc->provider); + rsc->provider = NULL; + } + if (NULL != rsc->params) { + free_str_table(rsc->params); + rsc->params = NULL; + } + if (rsc->last_op_table) { + g_hash_table_foreach_remove(rsc->last_op_table + , free_str_hash_pair, NULL); + g_hash_table_destroy(rsc->last_op_table); + rsc->last_op_table = NULL; + } + if (rsc->last_op_done) { + lrmd_op_destroy(rsc->last_op_done); + rsc->last_op_done = NULL; + } + + if (rsc->delay_timeout > 0) { + Gmain_timeout_remove(rsc->delay_timeout); + rsc->delay_timeout = (guint)0; + } + + free(rsc); + LRMAUDIT(); +} + +static lrmd_rsc_t* +lrmd_rsc_new(const char * id, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc; + rsc = (lrmd_rsc_t *)calloc(sizeof(lrmd_rsc_t),1); + if (rsc == NULL) { + lrmd_log(LOG_ERR, "%s: out of memory when calloc " + "a lrmd_rsc_t", __FUNCTION__); + return NULL; + } + rsc->delay_timeout = (guint)0; + if (id) { + rsc->id = strdup(id); + } + if (msg) { + rsc->type = strdup(ha_msg_value(msg, F_LRM_RTYPE)); + rsc->class = strdup(ha_msg_value(msg, F_LRM_RCLASS)); + if (NULL == ha_msg_value(msg, F_LRM_RPROVIDER)) { + lrmd_log(LOG_NOTICE, "%s(): No %s field in message" + , __FUNCTION__, F_LRM_RPROVIDER); + }else{ + rsc->provider = strdup(ha_msg_value(msg, F_LRM_RPROVIDER)); + if (rsc->provider == NULL) { + goto errout; + } + } + if (rsc->id == NULL + || rsc->type == NULL + || rsc->class == NULL) { + goto errout; + } + } + g_hash_table_insert(resources, strdup(id), rsc); + ++lrm_objectstats.rsccount; + return rsc; +errout: + lrmd_rsc_destroy(rsc); /* violated property */ /* Or so BEAM thinks :-) */ + rsc = NULL; + return rsc; +} + +static void +dump_op(gpointer key, gpointer val, gpointer data) +{ + lrmd_op_t* lrmd_op = (lrmd_op_t*) val; + + lrmd_op_dump(lrmd_op, "rsc->last_op_table"); +} +static void +dump_op_table(gpointer key, gpointer val, gpointer data) +{ + GHashTable* table = (GHashTable*) val; + + g_hash_table_foreach(table, dump_op, data); +} +static void +lrmd_rsc_dump(char* rsc_id, const char * text) +{ + static gboolean incall = FALSE; + GList* oplist; + lrmd_rsc_t* rsc=NULL; + + if( rsc_id ) { + rsc = lookup_rsc(rsc_id); + } else { + lrmd_debug(LOG_INFO + , "%s:%d: the rsc_id is NULL" + , __FUNCTION__, __LINE__); + return; + } + CHECK_ALLOCATED(rsc, "rsc", ); + if(!rsc) { + return; + } + + /* Avoid infinite recursion loops... */ + if (incall) { + return; + } + incall = TRUE; + /* TODO: Dump params and last_op_table FIXME */ + + lrmd_debug(LOG_DEBUG, "%s: BEGIN resource dump", text); + lrmd_debug(LOG_DEBUG, "%s: resource %s/%s/%s/%s" + , text + , lrm_str(rsc->id) + , lrm_str(rsc->type) + , lrm_str(rsc->class) + , lrm_str(rsc->provider)); + + lrmd_debug(LOG_DEBUG, "%s: rsc->op_list...", text); + for(oplist = g_list_first(rsc->op_list); oplist; + oplist = g_list_next(oplist)) { + lrmd_op_dump(oplist->data, "rsc->op_list"); + } + + lrmd_debug(LOG_DEBUG, "%s: rsc->repeat_op_list...", text); + for(oplist = g_list_first(rsc->repeat_op_list); oplist; + oplist=g_list_next(oplist)) { + lrmd_op_dump(oplist->data, "rsc->repeat_op_list"); + } + + if (rsc->last_op_done != NULL) { + lrmd_debug(LOG_DEBUG, "%s: rsc->last_op_done...", text); + lrmd_op_dump(rsc->last_op_done, "rsc->last_op_done"); + } + else { + lrmd_debug(LOG_DEBUG, "%s: rsc->last_op_done==NULL", text); + } + if (rsc->last_op_table) { + g_hash_table_foreach(rsc->last_op_table,dump_op_table,NULL); + } + else { + lrmd_debug(LOG_DEBUG, "%s: rsc->last_op_table==NULL", text); + } + lrmd_debug(LOG_DEBUG, "%s: END resource dump", text); + incall = FALSE; +}; +static void +dump_id_rsc_pair(gpointer key, gpointer value, gpointer user_data) +{ + char* rid = (char*)key; + char* text = (char*)user_data; + lrmd_rsc_dump(rid,text); +} +static void +lrmd_dump_all_resources(void) +{ + static gboolean incall = FALSE; + char text[]= "lrmd_dump_all_resources"; + if (incall) { + return; + } + incall = TRUE; + + lrmd_debug(LOG_DEBUG, "%d resources are managed by lrmd" + , g_hash_table_size(resources)); + g_hash_table_foreach(resources, dump_id_rsc_pair, text); + incall = FALSE; +} + + +#if 0 +static void +lrm_debug_running_op(lrmd_op_t* op, const char * text) +{ + char cmd[256]; + lrmd_op_dump(op, text); + CHECK_ALLOCATED(op, "op", ); + if (op->exec_pid >= 1) { + /* This really ought to use our logger + * So... it might not get forwarded to the central machine + * if you're testing with CTS -- FIXME!!! + */ + snprintf(cmd, sizeof(cmd) + , "ps -l -f -s %d | logger -p daemon.info -t 'T/O PS:'" + , op->exec_pid); + lrmd_debug(LOG_DEBUG, "Running [%s]", cmd); + if (system(cmd) != 0) { + lrmd_log(LOG_ERR, "Running [%s] failed", cmd); + } + snprintf(cmd, sizeof(cmd) + , "ps axww | logger -p daemon.info -t 't/o ps:'"); + lrmd_debug(LOG_DEBUG, "Running [%s]", cmd); + if (system(cmd) != 0) { + lrmd_log(LOG_ERR, "Running [%s] failed", cmd); + } + } +} +#endif +int +main(int argc, char ** argv) +{ + int req_restart = TRUE; + int req_status = FALSE; + int req_stop = FALSE; + + int argerr = 0; + int flag; + + while ((flag = getopt(argc, argv, OPTARGS)) != EOF) { + switch(flag) { + case 'h': /* Help message */ + usage(lrm_system_name, LSB_EXIT_OK); + break; + case 'v': /* Debug mode, more logs*/ + ++debug_level; + break; + case 's': /* Status */ + req_status = TRUE; + break; + case 'k': /* Stop (kill) */ + req_stop = TRUE; + break; + case 'r': /* Restart */ + req_restart = TRUE; + break; + /* Register to apphbd then monitored by it */ + case 'm': + reg_to_apphbd = TRUE; + break; + case 'i': /* Get apphb interval */ + if (optarg) { + apphb_interval = atoi(optarg); + } + break; + default: + ++argerr; + break; + } + } + + if (optind > argc) { + ++argerr; + } + + if (argerr) { + usage(lrm_system_name, LSB_EXIT_GENERIC); + } + + cl_log_set_entity(lrm_system_name); + cl_log_enable_stderr(debug_level?TRUE:FALSE); + cl_log_set_facility(HA_LOG_FACILITY); + + /* Use logd if it's enabled by heartbeat */ + cl_inherit_logging_environment(0); + + if (req_status){ + return init_status(PID_FILE, lrm_system_name); + } + + if (req_stop){ + return init_stop(PID_FILE); + } + + if (req_restart) { + init_stop(PID_FILE); + } + + return init_start(); +} + +int +init_status(const char *pid_file, const char *client_name) +{ + long pid = cl_read_pidfile(pid_file); + + if (pid > 0) { + fprintf(stderr, "%s is running [pid: %ld]\n" + , client_name, pid); + return LSB_STATUS_OK; + } + fprintf(stderr, "%s is stopped.\n", client_name); + return LSB_STATUS_STOPPED; +} + +int +init_stop(const char *pid_file) +{ + long pid; + int rc = LSB_EXIT_OK; + + + + if (pid_file == NULL) { + lrmd_log(LOG_ERR, "No pid file specified to kill process"); + return LSB_EXIT_GENERIC; + } + pid = cl_read_pidfile(pid_file); + + if (pid > 0) { + if (CL_KILL((pid_t)pid, SIGTERM) < 0) { + rc = (errno == EPERM + ? LSB_EXIT_EPERM : LSB_EXIT_GENERIC); + fprintf(stderr, "Cannot kill pid %ld\n", pid); + }else{ + lrmd_log(LOG_INFO, + "Signal sent to pid=%ld," + " waiting for process to exit", + pid); + + while (CL_PID_EXISTS(pid)) { + sleep(1); + } + } + } + return rc; +} + +static const char usagemsg[] = "[-srkhv]\n\ts: status\n\tr: restart" + "\n\tk: kill\n\tm: register to apphbd\n\ti: the interval of apphb\n\t" + "h: help\n\tv: debug\n"; + +void +usage(const char* cmd, int exit_status) +{ + FILE* stream; + + stream = exit_status ? stderr : stdout; + + fprintf(stream, "usage: %s %s", cmd, usagemsg); + fflush(stream); + + exit(exit_status); +} +/* + * In design, the lrmd should not know the meaning of operation type + * and the meaning of rc. This function is just for logging. + */ +static void +warning_on_active_rsc(gpointer key, gpointer value, gpointer user_data) +{ + int op_status, rc; + const char* op_type; + + lrmd_rsc_t* rsc = (lrmd_rsc_t*)value; + if (rsc->last_op_done != NULL) { + if (HA_OK != ha_msg_value_int(rsc->last_op_done->msg + , F_LRM_OPSTATUS, &op_status)) { + lrmd_debug(LOG_WARNING + ,"resource %s is left in UNKNOWN status." \ + "(last op done is damaged..)" + ,rsc->id); + return; + } + op_type = ha_msg_value(rsc->last_op_done->msg, F_LRM_OP); + if (op_status != LRM_OP_DONE) { + lrmd_debug(LOG_WARNING + ,"resource %s is left in UNKNOWN status." \ + "(last op %s finished without LRM_OP_DONE status.)" + ,rsc->id, op_type); + return; + } + if (HA_OK != ha_msg_value_int(rsc->last_op_done->msg + , F_LRM_RC, &rc)) { + lrmd_debug(LOG_WARNING + ,"resource %s is left in UNKNOWN status." \ + "(last op done is damaged..)" + ,rsc->id); + return; + } + if((rc == 0) && + (STRNCMP_CONST(op_type,"start") ==0 + ||STRNCMP_CONST(op_type,"monitor") ==0 + ||STRNCMP_CONST(op_type,"status") ==0)) { + lrmd_debug(LOG_WARNING + ,"resource %s is left in RUNNING status." \ + "(last op %s finished with rc 0.)" + ,rsc->id, op_type); + return; + } + if ((rc !=0 ) && + (STRNCMP_CONST(op_type,"start") ==0 + ||STRNCMP_CONST(op_type,"stop") ==0)) { + lrmd_debug(LOG_WARNING + ,"resource %s is left in UNKNOWN status." \ + "(last op %s finished with rc %d.)" + ,rsc->id, op_type, rc); + return; + } + } +} + +static gboolean +lrm_shutdown(void) +{ + lrmd_log(LOG_INFO,"lrmd is shutting down"); + if (mainloop != NULL && g_main_is_running(mainloop)) { + g_hash_table_foreach(resources, warning_on_active_rsc, NULL); + g_main_quit(mainloop); + }else { + exit(LSB_EXIT_OK); + } + return FALSE; +} +static void +has_pending_op(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_rsc_t* rsc = (lrmd_rsc_t*)value; + int* result = (int*)user_data; + if (rsc->op_list != NULL) { + *result = TRUE; + } +} +static gboolean +can_shutdown() +{ + int has_ops = FALSE; + g_hash_table_foreach(resources, has_pending_op, &has_ops); + + return !has_ops; +} +gboolean +sigterm_action(int nsig, gpointer user_data) +{ + shutdown_in_progress = TRUE; + + if (can_shutdown()) { + lrm_shutdown(); + } else { + lrmd_log(LOG_INFO, "sigterm_action: shutdown postponed, some operations are still running"); + } + return TRUE; +} + +static void +register_pid(gboolean do_fork, + gboolean (*shutdown)(int nsig, gpointer userdata)) +{ + int j; + + umask(022); + + for (j=0; j < 3; ++j) { + close(j); + (void)open("/dev/null", j == 0 ? O_RDONLY : O_WRONLY); + } + CL_IGNORE_SIG(SIGINT); + CL_IGNORE_SIG(SIGHUP); + CL_DEFAULT_SIG(SIGPIPE); + G_main_add_SignalHandler(G_PRIORITY_HIGH, SIGTERM + , shutdown, NULL, NULL); + cl_signal_set_interrupt(SIGTERM, 1); + cl_signal_set_interrupt(SIGCHLD, 1); + /* At least they are harmless, I think. ;-) */ + cl_signal_set_interrupt(SIGINT, 0); + cl_signal_set_interrupt(SIGHUP, 0); +} + +static int +init_using_apphb(void) +{ +#ifdef ENABLE_APPHB + char lrmd_instance[40]; + + if (reg_to_apphbd == FALSE) { + return -1; + } + + snprintf(lrmd_instance, sizeof(lrmd_instance), "%s_%ld" + , lrm_system_name, (long)getpid()); + if (apphb_register(lrm_system_name, lrmd_instance) != 0) { + lrmd_log(LOG_ERR, "Failed when trying to register to apphbd."); + lrmd_log(LOG_ERR, "Maybe apphbd is not running. Quit."); + return -1; + } + lrmd_log(LOG_INFO, "Registered to apphbd."); + + apphb_setinterval(apphb_interval); + apphb_setwarn(apphb_interval*APPHB_WARNTIME_FACTOR); + + Gmain_timeout_add(apphb_interval - APPHB_INTVL_DETLA, emit_apphb, NULL); +#endif + return 0; +} + +static gboolean +emit_apphb(gpointer data) +{ +#ifdef ENABLE_APPHB + if (reg_to_apphbd == FALSE) { + return FALSE; + } + + if (apphb_hb() != 0) { + lrmd_log(LOG_ERR, "emit_apphb: Failed to emit an apphb."); + reg_to_apphbd = FALSE; + return FALSE; + }; +#endif + return TRUE; +} + +static void +calc_max_children() +{ +#ifdef _SC_NPROCESSORS_ONLN + int nprocs; + + nprocs = sysconf(_SC_NPROCESSORS_ONLN); + if( nprocs < 1 ) { + lrmd_log(LOG_WARNING, "%s: couldn't get the number of processors" + , __FUNCTION__); + } else { + if( nprocs/2 > max_child_count ) { + max_child_count = nprocs/2; + } + lrmd_log(LOG_INFO, "max-children set to %d " + "(%d processors online)", max_child_count, nprocs); + return; + } +#else + lrmd_log(LOG_WARNING, "%s: cannot get the number of processors " + "on this platform", __FUNCTION__); +#endif + lrmd_log(LOG_INFO, "max-children set to %d", max_child_count); +} + +/* main loop of the daemon*/ +int +init_start () +{ + DIR* dir = NULL; + PILPluginUniv * PluginLoadingSystem = NULL; + struct dirent* subdir; + char* dot = NULL; + char* ra_name = NULL; + int len; + IPC_WaitConnection* conn_cmd = NULL; + IPC_WaitConnection* conn_cbk = NULL; + + GHashTable* conn_cmd_attrs; + GHashTable* conn_cbk_attrs; + + char path[] = IPC_PATH_ATTR; + char cmd_path[] = LRM_CMDPATH; + char cbk_path[] = LRM_CALLBACKPATH; + + PILGenericIfMgmtRqst RegisterRqsts[]= { + {"RAExec", &RAExecFuncs, NULL, NULL, NULL}, + { NULL, NULL, NULL, NULL, NULL} }; + + if( getenv("LRMD_MAX_CHILDREN") ) { + set_lrmd_param("max-children", getenv("LRMD_MAX_CHILDREN")); + } else { + calc_max_children(); + } + + qsort(msg_maps, MSG_NR, sizeof(struct msg_map), msg_type_cmp); + + if (cl_lock_pidfile(PID_FILE) < 0) { + lrmd_log(LOG_ERR, "already running: [pid %d].", cl_read_pidfile(PID_FILE)); + lrmd_log(LOG_ERR, "Startup aborted (already running). Shutting down."); + exit(100); + } + + register_pid(FALSE, sigterm_action); + + /* load RA plugins */ + PluginLoadingSystem = NewPILPluginUniv (HA_PLUGIN_DIR); + PILLoadPlugin(PluginLoadingSystem, "InterfaceMgr", "generic", + &RegisterRqsts); + + /* + * FIXME!!! + * Much of the code through the end of the next loop is + * unnecessary - The plugin system will do this for you quite + * nicely. And, it does it portably, too... + */ + + dir = opendir(LRM_PLUGIN_DIR); + if (NULL == dir) { + lrmd_log(LOG_ERR, "main: can not open RA plugin dir "LRM_PLUGIN_DIR); + lrmd_log(LOG_ERR, "Startup aborted (no RA plugin). Shutting down."); + exit(100); + } + + while ( NULL != (subdir = readdir(dir))) { + /* skip . and .. */ + if ( '.' == subdir->d_name[0]) { + continue; + } + /* skip the other type files */ + if (NULL == strstr(subdir->d_name, ".so")) { + continue; + } + /* remove the ".so" */ + dot = strchr(subdir->d_name,'.'); + if (NULL != dot) { + len = (int)(dot - subdir->d_name); + ra_name = g_strndup(subdir->d_name,len); + } + else { + ra_name = g_strdup(subdir->d_name); + } + PILLoadPlugin(PluginLoadingSystem , "RAExec", ra_name, NULL); + ra_class_list = g_list_append(ra_class_list,ra_name); + } + closedir(dir); dir = NULL; /* Don't forget to close 'dir' */ + + /* + *create the waiting connections + *one for register the client, + *the other is for create the callback channel + */ + + /*Create a waiting connection to accept command connect from client*/ + conn_cmd_attrs = g_hash_table_new(g_str_hash, g_str_equal); + g_hash_table_insert(conn_cmd_attrs, path, cmd_path); + conn_cmd = ipc_wait_conn_constructor(IPC_ANYTYPE, conn_cmd_attrs); + g_hash_table_destroy(conn_cmd_attrs); + if (NULL == conn_cmd) { + lrmd_log(LOG_ERR, + "main: can not create wait connection for command."); + lrmd_log(LOG_ERR, "Startup aborted (can't create comm channel). Shutting down."); + + exit(100); + } + + /*Create a source to handle new connect rquests for command*/ + G_main_add_IPC_WaitConnection( G_PRIORITY_HIGH, conn_cmd, NULL, FALSE, + on_connect_cmd, conn_cmd, NULL); + + /* auth is static, but used when clients register */ + auth = ipc_str_to_auth(ADMIN_UIDS, strlen(ADMIN_UIDS), "", 0); + + /* + * Create a waiting connection to accept the callback connect from client + */ + conn_cbk_attrs = g_hash_table_new(g_str_hash, g_str_equal); + g_hash_table_insert(conn_cbk_attrs, path, cbk_path); + conn_cbk = ipc_wait_conn_constructor( IPC_ANYTYPE, conn_cbk_attrs); + g_hash_table_destroy(conn_cbk_attrs); + + if (NULL == conn_cbk) { + lrmd_log(LOG_ERR, + "main: can not create wait connection for callback."); + lrmd_log(LOG_ERR, "Startup aborted (can't create comm channel). Shutting down."); + exit(100); + } + + /*Create a source to handle new connect rquests for callback*/ + G_main_add_IPC_WaitConnection( G_PRIORITY_HIGH, conn_cbk, NULL, FALSE, + on_connect_cbk, conn_cbk, NULL); + + /* our child signal handling involves calls with + * unpredictable timing; so we raise the limit to + * reduce the number of warnings + */ + set_sigchld_proctrack(G_PRIORITY_HIGH,10*DEFAULT_MAXDISPATCHTIME); + + lrmd_log(LOG_INFO, "enabling coredumps"); + /* Although lrmd can count on the parent to enable coredump, still + * set it here for test, when start manually. + */ + cl_cdtocoredir(); + cl_enable_coredumps(TRUE); + + /* Allow us to always take a "secure" core dump + * We might have STONITH logins and passwords, etc. in our address + * space - so we need to make sure it's only readable by root. + * Calling this function accomplishes that. + */ + cl_set_all_coredump_signal_handlers(); + if( drop_privs(0, 0) ) { /* become "nobody" */ + lrmd_log(LOG_WARNING,"%s: failed to drop privileges: %s" + , __FUNCTION__, strerror(errno)); + } + + /* + * Add the signal handler for SIGUSR1, SIGUSR2. + * They are used to change the debug level. + */ + G_main_add_SignalHandler(G_PRIORITY_HIGH, SIGUSR1, + debug_level_adjust, NULL, NULL); + G_main_add_SignalHandler(G_PRIORITY_HIGH, SIGUSR2, + debug_level_adjust, NULL, NULL); + + /* + * alloc memory for client table and resource table + */ + clients = g_hash_table_new(g_int_hash, g_int_equal); + if (clients == NULL) { + cl_log(LOG_ERR, "can not new hash table clients"); + exit(100); + } + resources = g_hash_table_new_full(g_str_hash + , g_str_equal, free, NULL); + if (resources == NULL) { + cl_log(LOG_ERR, "can not new hash table resources"); + exit(100); + } + + /*Create the mainloop and run it*/ + mainloop = g_main_new(FALSE); + lrmd_debug(LOG_DEBUG, "main: run the loop..."); + lrmd_log(LOG_INFO, "Started."); + + /* apphb initializing */ + init_using_apphb(); + emit_apphb(NULL); /* Avoid warning */ + + g_main_run(mainloop); + + emit_apphb(NULL); + if (reg_to_apphbd == TRUE) { +#ifdef ENABLE_APPHB + apphb_unregister(); +#endif + reg_to_apphbd = FALSE; + } + + if( return_to_orig_privs() ) { + cl_perror("%s: failed to raise privileges", __FUNCTION__); + } + conn_cmd->ops->destroy(conn_cmd); + conn_cmd = NULL; + + conn_cbk->ops->destroy(conn_cbk); + conn_cbk = NULL; + + ipc_destroy_auth(auth); + if (cl_unlock_pidfile(PID_FILE) == 0) { + lrmd_debug(LOG_DEBUG, "[%s] stopped", lrm_system_name); + } + return 0; +} + +/* + *GLoop Message Handlers + */ +gboolean +on_connect_cmd (IPC_Channel* ch, gpointer user_data) +{ + lrmd_client_t* client = NULL; + + /* check paremeters */ + if (NULL == ch) { + lrmd_log(LOG_ERR, "on_connect_cmd: channel is null"); + return TRUE; + } + /* create new client */ + /* the register will be finished in on_msg_register */ + client = lrmd_client_new(); + if (client == NULL) { + return TRUE; + } + client->app_name = NULL; + client->ch_cmd = ch; + client->g_src = G_main_add_IPC_Channel(G_PRIORITY_DEFAULT, + ch, FALSE, on_receive_cmd, (gpointer)client, + on_remove_client); + + + return TRUE; +} + +gboolean +on_connect_cbk (IPC_Channel* ch, gpointer user_data) +{ + /*client connect for create the second channel for call back*/ + pid_t pid; + const char* type = NULL; + struct ha_msg* msg = NULL; + lrmd_client_t* client = NULL; + + if (NULL == ch) { + lrmd_log(LOG_ERR, "on_connect_cbk: channel is null"); + return TRUE; + } + + /* Isn't this kind of a tight timing assumption ?? + * This operation is non-blocking -- IIRC + * Maybe this should be moved to the input dispatch function + * for this channel when we make a GSource from it. + * FIXME + */ + + /*get the message, ends up in socket_waitin */ + msg = msgfromIPC_noauth(ch); + if (NULL == msg) { + lrmd_log(LOG_ERR, "on_connect_cbk: can not receive msg"); + return TRUE; + } + + /*check if it is a register message*/ + type = ha_msg_value(msg, F_LRM_TYPE); + if (0 != STRNCMP_CONST(type, REGISTER)) { + lrmd_log(LOG_ERR, "on_connect_cbk: received a message which is " + "not known by lrmd."); + ha_msg_del(msg); + send_ret_msg(ch, HA_FAIL); + return TRUE; + } + + /*get the pid of client */ + if (HA_OK != ha_msg_value_int(msg, F_LRM_PID, &pid)) { + lrmd_log(LOG_ERR, "on_connect_cbk: can not get pid from the " + "message."); + ha_msg_del(msg); + send_ret_msg(ch, HA_FAIL); + return TRUE; + } + ha_msg_del(msg); + + /*get the client in the client list*/ + client = lookup_client(pid); + if (NULL == client) { + lrmd_log(LOG_ERR, "on_connect_cbk: donnot find the client " + "[pid:%d] in internal client list. ", pid); + send_ret_msg(ch, HA_FAIL); + return TRUE; + } + if (client->ch_cbk != NULL) { + client->ch_cbk->ops->destroy(client->ch_cbk); + client->ch_cbk = NULL; + } + client->g_src_cbk = G_main_add_IPC_Channel(G_PRIORITY_DEFAULT + , ch, FALSE,NULL,NULL,NULL); + + /*fill the channel of callback field*/ + client->ch_cbk = ch; + send_ret_msg(ch, HA_OK); + return TRUE; +} + +int +msg_type_cmp(const void *p1, const void *p2) +{ + + return strncmp( + ((const struct msg_map *)p1)->msg_type, + ((const struct msg_map *)p2)->msg_type, + MAX_MSGTYPELEN); +} + +gboolean +on_receive_cmd (IPC_Channel* ch, gpointer user_data) +{ + struct msg_map *msgmap_p, in_type; + lrmd_client_t* client = NULL; + struct ha_msg* msg = NULL; + char *msg_s; + int ret = FALSE; + + client = (lrmd_client_t*)user_data; + + if (IPC_DISCONNECT == ch->ch_status) { + lrmd_debug(LOG_DEBUG, + "on_receive_cmd: the IPC to client [pid:%d] disconnected." + , client->pid); + return FALSE; + } + + if (!ch->ops->is_message_pending(ch)) { + lrmd_debug(LOG_DEBUG, "on_receive_cmd: no pending message in IPC " + "channel."); + return TRUE; + } + + + /*get the message */ + msg = msgfromIPC(ch, 0); + if (NULL == msg) { + lrmd_log(LOG_ERR, "on_receive_cmd: can not receive messages."); + return TRUE; + } + + if (TRUE == shutdown_in_progress ) { + send_ret_msg(ch,HA_FAIL); + ha_msg_del(msg); + lrmd_log(LOG_INFO, "%s: new requests denied," \ + " we're about to shutdown", __FUNCTION__); + return TRUE; + } + + /*dispatch the message*/ + in_type.msg_type = ha_msg_value(msg, F_LRM_TYPE); + if( !in_type.msg_type ) { + LOG_FAILED_TO_GET_FIELD(F_LRM_TYPE); + ha_msg_del(msg); + return TRUE; + } + msg_s = msg2string(msg); + if( msg_s ) { + lrmd_debug2(LOG_DEBUG,"dumping request: %s",msg_s); + free(msg_s); + } + + if (!(msgmap_p = bsearch(&in_type, msg_maps, + MSG_NR, sizeof(struct msg_map), msg_type_cmp) + )) { + + lrmd_log(LOG_ERR, "on_receive_cmd: received an unknown msg"); + } else { + if( !client->app_name && msgmap_p->handler != on_msg_register ) { + ha_msg_del(msg); + lrmd_log(LOG_ERR, "%s: the client needs to register first", __FUNCTION__); + return FALSE; + } + + if( client->priv_lvl < msgmap_p->min_priv ) { + ha_msg_del(msg); + lrmd_log(LOG_ERR, "%s: insufficient privileges for %s (pid %d)" + , __FUNCTION__ + , client->app_name, client->pid); + return FALSE; + } + strncpy(client->lastrequest, in_type.msg_type, sizeof(client->lastrequest)); + client->lastrequest[sizeof(client->lastrequest)-1]='\0'; + client->lastreqstart = time(NULL); + /*call the handler of the message*/ + ret = msgmap_p->handler(client, msg); + client->lastreqend = time(NULL); + + /*return rc to client if need*/ + if (send_msg_now(msgmap_p)) { + send_ret_msg(ch, ret); + client->lastrcsent = time(NULL); + } + } + + /*delete the msg*/ + ha_msg_del(msg); + + return ret; +} +static void +remove_repeat_op_from_client(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_rsc_t* rsc = (lrmd_rsc_t*)value; + pid_t pid = GPOINTER_TO_UINT(user_data); /* pointer cast as int */ + + (void)flush_all(&(rsc->repeat_op_list),pid); +} + +/* Remove all direct pointer references to 'client' before destroying it */ +static int +unregister_client(lrmd_client_t* client) +{ + CHECK_ALLOCATED(client, "client", HA_FAIL); + + if (NULL == lookup_client(client->pid)) { + lrmd_log(LOG_ERR,"%s: can not find client %s [pid %d] when try " + "to unregister it." + , __FUNCTION__ + , client->app_name, client->pid); + return HA_FAIL; + } + + /* Search all resources for repeating ops this client owns */ + g_hash_table_foreach(resources + , remove_repeat_op_from_client, GUINT_TO_POINTER(client->pid)); + + /* Remove from clients */ + g_hash_table_remove(clients, (gpointer)&client->pid); + + lrmd_debug(LOG_DEBUG, "%s: client %s [pid:%d] is unregistered" + , __FUNCTION__ + , client->app_name + , client->pid); + return HA_OK; +} + +void +on_remove_client (gpointer user_data) +{ + lrmd_client_t* client = (lrmd_client_t*) user_data; + + CHECK_ALLOCATED(client, "client", ); + if (client->g_src != NULL) { + G_main_del_IPC_Channel(client->g_src); + } + if (client->g_src_cbk != NULL) { + G_main_del_IPC_Channel(client->g_src_cbk); + } + lrmd_client_destroy(client); + +} + + +/* This function called when its time to run a repeating operation now */ +/* Move op from repeat queue to running queue */ +gboolean +on_repeat_op_readytorun(gpointer data) +{ + lrmd_op_t* op = NULL; + lrmd_rsc_t* rsc = NULL; + + LRMAUDIT(); + op = (lrmd_op_t*)data; + CHECK_ALLOCATED(op, "op", FALSE ); + + if (op->exec_pid == 0) { + lrmd_log(LOG_ERR, "%s: exec_pid is 0 (internal error)" + , __FUNCTION__); + return FALSE; + } + + lrmd_debug2(LOG_DEBUG + , "%s: remove operation %s from the repeat operation list and " + "add it to the operation list" + , __FUNCTION__, op_info(op)); + + if( op->rsc_id ) { + rsc = lookup_rsc(op->rsc_id); + } else { + lrmd_debug(LOG_INFO + , "%s: the rsc_id in op %s is NULL" + , __FUNCTION__, op_info(op)); + return FALSE; + } + + rsc->repeat_op_list = g_list_remove(rsc->repeat_op_list, op); + if (op->repeat_timeout_tag != 0) { + op->repeat_timeout_tag = (guint)0; + } + + op->exec_pid = -1; + + if (!shutdown_in_progress) { + add_op_to_runlist(rsc,op); + } + perform_op(rsc); + + LRMAUDIT(); + return FALSE; +} + +/*LRM Message Handlers*/ +int +on_msg_register(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_client_t* exist = NULL; + const char* app_name = NULL; + + CHECK_ALLOCATED(msg, "register message", HA_FAIL); + + app_name = ha_msg_value(msg, F_LRM_APP); + if (NULL == app_name) { + lrmd_log(LOG_ERR, "on_msg_register: no app_name in " + "the ha message."); + return HA_FAIL; + } + client->app_name = strdup(app_name); + + return_on_no_int_value(msg, F_LRM_PID, &client->pid); + return_on_no_int_value(msg, F_LRM_GID, (int *)&client->gid); + return_on_no_int_value(msg, F_LRM_UID, (int *)&client->uid); + + exist = lookup_client(client->pid); + if (NULL != exist) { + g_hash_table_remove(clients, (gpointer)&client->pid); + on_remove_client(exist); + lrmd_log(LOG_NOTICE, + "on_msg_register: the client [pid:%d] already exists in " + "internal client list, let remove it at first." + , client->pid); + } + + /* everybody can connect, but only certain UIDs can perform + * administrative actions + */ + if( client->ch_cmd->ops->verify_auth(client->ch_cmd, auth) == IPC_OK ) + client->priv_lvl = PRIV_ADMIN; + else + client->priv_lvl = 0; + + g_hash_table_insert(clients, (gpointer)&client->pid, client); + lrmd_debug(LOG_DEBUG, "on_msg_register:client %s [%d] registered" + , client->app_name + , client->pid); + + return HA_OK; +} + +int +on_msg_get_rsc_classes(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + lrmd_debug2(LOG_DEBUG + , "on_msg_get_rsc_classes:client [%d] wants to get rsc classes" + , client->pid); + + ret = create_lrm_ret(HA_OK, 4); + CHECK_RETURN_OF_CREATE_LRM_RET; + + cl_msg_add_list(ret,F_LRM_RCLASS,ra_class_list); + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_rsc_classes: cannot send the ret mesage"); + } + ha_msg_del(ret); + + return HA_OK; +} + +int +on_msg_get_rsc_types(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + struct RAExecOps * RAExec = NULL; + GList* types = NULL; + GList* type; + const char* rclass = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + ret = create_lrm_ret(HA_OK,5); + CHECK_RETURN_OF_CREATE_LRM_RET; + + rclass = ha_msg_value(msg, F_LRM_RCLASS); + if (rclass == NULL) { + lrmd_log(LOG_ERR, "on_msg_get_rsc_types: cannot get the " + "resource class field from the message."); + send_ret_msg(client->ch_cmd, HA_FAIL); + return HA_FAIL; + } + + lrmd_debug2(LOG_DEBUG, "on_msg_get_rsc_types: the client [pid:%d] " + "wants to get resource types of resource class %s" + , client->pid, rclass); + + RAExec = g_hash_table_lookup(RAExecFuncs,rclass); + + if (NULL == RAExec) { + lrmd_log(LOG_NOTICE, "on_msg_get_rsc_types: can not find this " + "RA class %s.", rclass); + } else { + if (0 <= RAExec->get_resource_list(&types) && types != NULL) { + cl_msg_add_list(ret, F_LRM_RTYPES, types); + while (NULL != (type = g_list_first(types))) { + types = g_list_remove_link(types, type); + g_free(type->data); + g_list_free_1(type); + } + g_list_free(types); + } + } + + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_rsc_types: can not send the ret message."); + } + ha_msg_del(ret); + + return HA_OK; +} + +int +on_msg_get_rsc_providers(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + struct RAExecOps * RAExec = NULL; + GList* providers = NULL; + GList* provider = NULL; + const char* rclass = NULL; + const char* rtype = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + ret = create_lrm_ret(HA_OK,5); + CHECK_RETURN_OF_CREATE_LRM_RET; + + rclass = ha_msg_value(msg, F_LRM_RCLASS); + rtype = ha_msg_value(msg, F_LRM_RTYPE); + if( !rclass || !rtype ) { + lrmd_log(LOG_NOTICE + , "%s: could not retrieve resource class or type" + , __FUNCTION__); + send_ret_msg(client->ch_cmd, HA_FAIL); + return HA_FAIL; + } + + lrmd_debug2(LOG_DEBUG + , "%s: the client [%d] wants to get rsc privider of %s::%s" + , __FUNCTION__ + , client->pid + , rclass + , rtype); + + RAExec = g_hash_table_lookup(RAExecFuncs, rclass); + + if (NULL == RAExec) { + lrmd_log(LOG_NOTICE + , "%s: can not find the class %s." + , __FUNCTION__ + , rclass); + } + else { + if (0 <= RAExec->get_provider_list(rtype, &providers)) { + if (providers != NULL) { + cl_msg_add_list(ret, F_LRM_RPROVIDERS, providers); + } + while (NULL != (provider = g_list_first(providers))) { + providers = g_list_remove_link(providers, provider); + g_free(provider->data); + g_list_free_1(provider); + } + g_list_free(providers); + } + } + + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_rsc_providers: can not send the ret msg"); + } + ha_msg_del(ret); + + return HA_OK; +} + +int +on_msg_get_metadata(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + struct RAExecOps * RAExec = NULL; + const char* rtype = NULL; + const char* rclass = NULL; + const char* provider = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + rtype = ha_msg_value(msg, F_LRM_RTYPE); + rclass = ha_msg_value(msg, F_LRM_RCLASS); + provider = ha_msg_value(msg, F_LRM_RPROVIDER); + + lrmd_debug2(LOG_DEBUG + , "%s: the client [pid:%d] wants to get rsc metadata of %s::%s::%s." + , __FUNCTION__ + , client->pid + , lrm_str(rclass) + , lrm_str(provider) + , lrm_str(rtype)); + + ret = create_lrm_ret(HA_OK, 5); + CHECK_RETURN_OF_CREATE_LRM_RET; + + RAExec = g_hash_table_lookup(RAExecFuncs,rclass); + if (NULL == RAExec) { + lrmd_log(LOG_NOTICE + , "%s: can not find the class %s." + , __FUNCTION__ + , rclass); + } + else { + char* meta = RAExec->get_resource_meta(rtype,provider); + if (NULL != meta && strlen(meta) > 0) { + if (HA_OK != ha_msg_add(ret,F_LRM_METADATA, meta)) { + LOG_FAILED_TO_ADD_FIELD("metadata"); + } + g_free(meta); + } + else { + lrmd_log(LOG_WARNING + , "%s: empty metadata for %s::%s::%s." + , __FUNCTION__ + , lrm_str(rclass) + , lrm_str(provider) + , lrm_str(rtype)); + ha_msg_mod_int(ret, F_LRM_RET, HA_FAIL); + } + } + + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_metadata: can not send the ret msg"); + } + ha_msg_del(ret); + + return HA_OK; +} +static void +add_rid_to_msg(gpointer key, gpointer value, gpointer user_data) +{ + char* rid = (char*)key; + struct ha_msg* msg = (struct ha_msg*)user_data; + if (HA_OK != cl_msg_list_add_string(msg,F_LRM_RID,rid)) { + LOG_FAILED_TO_ADD_FIELD("resource id"); + } +} +int +on_msg_get_all(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + lrmd_debug2(LOG_DEBUG + , "on_msg_get_all:client [%d] want to get all rsc information." + , client->pid); + + ret = create_lrm_ret(HA_OK, g_hash_table_size(resources) + 1); + CHECK_RETURN_OF_CREATE_LRM_RET; + + g_hash_table_foreach(resources, add_rid_to_msg, ret); + + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, "on_msg_get_all: can not send the ret msg"); + } + ha_msg_del(ret); + + return HA_OK; +} +int +on_msg_get_rsc(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + lrmd_rsc_t* rsc = NULL; + const char* id = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + id = ha_msg_value(msg, F_LRM_RID); + + lrmd_debug2(LOG_DEBUG + , "on_msg_get_rsc: the client [pid:%d] wants to get " + "the information of the resource [rsc_id: %s]" + , client->pid, lrmd_nullcheck(id)); + + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_debug2(LOG_DEBUG + , "on_msg_get_rsc: no rsc with id %s." + , lrmd_nullcheck(id)); + ret = create_lrm_ret(HA_FAIL, 1); + CHECK_RETURN_OF_CREATE_LRM_RET; + } + else { + ret = create_lrm_ret(HA_OK, 5); + CHECK_RETURN_OF_CREATE_LRM_RET; + + if (HA_OK != ha_msg_add(ret, F_LRM_RID, rsc->id) + || HA_OK != ha_msg_add(ret, F_LRM_RTYPE, rsc->type) + || HA_OK != ha_msg_add(ret, F_LRM_RCLASS, rsc->class)) { + ha_msg_del(ret); + lrmd_log(LOG_ERR, + "on_msg_get_rsc: failed to add fields to msg."); + return HA_FAIL; + } + if( rsc->provider ) { + if (HA_OK != ha_msg_add(ret, F_LRM_RPROVIDER, + rsc->provider)) { + ha_msg_del(ret); + LOG_FAILED_TO_ADD_FIELD("provider"); + return HA_FAIL; + } + } + + if ( rsc->params && + HA_OK!=ha_msg_add_str_table(ret,F_LRM_PARAM,rsc->params)) { + ha_msg_del(ret); + LOG_FAILED_TO_ADD_FIELD("parameter"); + return HA_FAIL; + } + + } + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, "on_msg_get_rsc: can not send the ret msg"); + } + ha_msg_del(ret); + + return HA_OK; +} + +int +on_msg_get_last_op(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + const char* op_type = NULL; + lrmd_rsc_t* rsc = NULL; + const char* rid = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + rid = ha_msg_value(msg, F_LRM_RID); + op_type = ha_msg_value(msg, F_LRM_OP); + + lrmd_debug2(LOG_DEBUG + , "on_msg_get_last_op:client %s[%d] want to get the information " + "regarding last %s op on %s" + , client->app_name, client->pid + , lrmd_nullcheck(op_type), lrmd_nullcheck(rid)); + + rsc = lookup_rsc_by_msg(msg); + if (NULL != rsc && NULL != op_type) { + GHashTable* table = g_hash_table_lookup(rsc->last_op_table + , client->app_name); + if (NULL != table ) { + lrmd_op_t* op = g_hash_table_lookup(table, op_type); + if (NULL != op) { + lrmd_debug(LOG_DEBUG + , "%s: will return op %s" + , __FUNCTION__ + , op_type); + + ret = op_to_msg(op); + if (NULL == ret) { + lrmd_log(LOG_ERR + , "%s: can't create a message with op_to_msg." + , __FUNCTION__); + + } else + if (HA_OK != ha_msg_add_int(ret + , F_LRM_OPCNT, 1)) { + LOG_FAILED_TO_ADD_FIELD("operation count"); + } + } + } + } + + if (NULL == ret) { + lrmd_log(LOG_ERR + , "%s: return ha_msg ret is null, will re-create it again." + , __FUNCTION__); + ret = create_lrm_ret(HA_OK, 1); + CHECK_RETURN_OF_CREATE_LRM_RET; + + if (HA_OK != ha_msg_add_int(ret, F_LRM_OPCNT, 0)) { + LOG_FAILED_TO_ADD_FIELD("operation count"); + } + + } + + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, "on_msg_get_last_op: can not send the ret msg"); + } + ha_msg_del(ret); + + return HA_OK; +} + +int +on_msg_del_rsc(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc = NULL; + const char* id = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + id = ha_msg_value(msg, F_LRM_RID); + lrmd_debug2(LOG_DEBUG + , "%s: client [%d] wants to delete rsc %s" + , __FUNCTION__, client->pid, lrmd_nullcheck(id)); + + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_log(LOG_ERR, "%s: no rsc with id %s.",__FUNCTION__,id); + return -1; + } + LRMAUDIT(); + (void)flush_all(&(rsc->repeat_op_list),0); + if( flush_all(&(rsc->op_list),0) ) { + set_rsc_removal_pending(rsc); + lrmd_log(LOG_INFO, "resource %s busy, removal pending", rsc->id); + LRMAUDIT(); + return HA_RSCBUSY; /* resource is busy, removal delayed */ + } + lrmd_rsc_destroy(rsc); + LRMAUDIT(); + return HA_OK; +} + +static int +prepare_failmsg(struct ha_msg* msg, int fail_rc, const char *fail_reason) +{ + call_id++; /* use the next id */ + if (HA_OK != ha_msg_mod(msg,F_LRM_OP,ASYNC_OP_NAME) + || HA_OK != ha_msg_add(msg,F_LRM_FAIL_REASON,fail_reason) + || HA_OK != ha_msg_mod_int(msg,F_LRM_ASYNCMON_RC,fail_rc) + || HA_OK != ha_msg_mod_int(msg,F_LRM_RC,fail_rc) + || HA_OK != ha_msg_mod_int(msg,F_LRM_OPSTATUS,(int)LRM_OP_DONE) + || HA_OK != ha_msg_mod_int(msg,F_LRM_CALLID,call_id) + || HA_OK != ha_msg_mod_int(msg,F_LRM_TIMEOUT,0) + || HA_OK != ha_msg_mod_int(msg,F_LRM_INTERVAL,0) + || HA_OK != ha_msg_mod_int(msg,F_LRM_TARGETRC,EVERYTIME) + || HA_OK != ha_msg_mod_int(msg,F_LRM_DELAY,0) + ) { + lrmd_log(LOG_ERR,"%s:%d: cannot add field to a message" + , __FUNCTION__, __LINE__); + return 1; + } + return 0; +} + +static void +async_notify(gpointer key, gpointer val, gpointer data) +{ + struct ha_msg* msg = (struct ha_msg*)data; + lrmd_client_t* client; + + client = lookup_client_by_name((char *)key); + if (!client) { + lrmd_log(LOG_INFO, + "%s: client %s not found, probably signed out", __FUNCTION__, (char *)key); + } else { + send_msg(msg, client); + } +} + +int +on_msg_fail_rsc(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc; + const char* id; + int fail_rc = -1; + const char *fail_reason; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + id = ha_msg_value(msg, F_LRM_RID); + lrmd_debug2(LOG_DEBUG + , "%s: client [%d] wants to fail rsc %s" + , __FUNCTION__, client->pid, lrmd_nullcheck(id)); + + rsc = lookup_rsc_by_msg(msg); + if (!rsc) { + lrmd_log(LOG_ERR, "%s: no resource with id %s." + , __FUNCTION__, lrmd_nullcheck(id)); + return HA_FAIL; + } + fail_reason = ha_msg_value(msg,F_LRM_FAIL_REASON); + if (!fail_reason || *fail_reason == '\0') { + fail_reason = DEFAULT_FAIL_REASON; + } + if (HA_OK != ha_msg_value_int(msg,F_LRM_ASYNCMON_RC,&fail_rc) || fail_rc <= 0) { + fail_rc = DEFAULT_FAIL_RC; + } + if (prepare_failmsg(msg,fail_rc,fail_reason)) + return HA_FAIL; + lrmd_log(LOG_WARNING + , "received asynchronous failure for rsc %s (rc: %d, reason: %s)" + , lrmd_nullcheck(id), fail_rc, fail_reason); + /* notify all clients from last_op table about the failure */ + if (rsc->last_op_table) { + g_hash_table_foreach(rsc->last_op_table,async_notify,msg); + } else { + lrmd_log(LOG_INFO + , "rsc to be failed %s had no operations so far", lrmd_nullcheck(id)); + send_msg(msg, client); + } + return HA_OK; +} + +static gboolean +free_str_hash_pair(gpointer key, gpointer value, gpointer user_data) +{ + GHashTable* table = (GHashTable*) value; + free(key); + g_hash_table_foreach_remove(table, free_str_op_pair, NULL); + g_hash_table_destroy(table); + return TRUE; +} + +static gboolean +free_str_op_pair(gpointer key, gpointer value, gpointer user_data) +{ + lrmd_op_t* op = (lrmd_op_t*)value; + + if (NULL == op) { + lrmd_log(LOG_ERR, "%s(): NULL op in op_pair(%s)" , __FUNCTION__ + , (const char *)key); + }else{ + lrmd_op_destroy(op); + } + return TRUE; +} + +int +on_msg_add_rsc(lrmd_client_t* client, struct ha_msg* msg) +{ + GList* node; + gboolean ra_type_exist = FALSE; + char* class = NULL; + lrmd_rsc_t* rsc = NULL; + const char* id = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + return_on_no_value(msg, F_LRM_RID,id); + + lrmd_debug(LOG_DEBUG + , "on_msg_add_rsc:client [%d] adds resource %s" + , client->pid, lrmd_nullcheck(id)); + + if (RID_LEN <= strlen(id)) { + lrmd_log(LOG_ERR, "on_msg_add_rsc: rsc_id is too long."); + return HA_FAIL; + } + + if (NULL != lookup_rsc(id)) { + lrmd_log(LOG_ERR, "on_msg_add_rsc: same id resource exists."); + return HA_FAIL; + } + + LRMAUDIT(); + rsc = lrmd_rsc_new(id, msg); + if (rsc == NULL) { + return HA_FAIL; + } + + ra_type_exist = FALSE; + for(node=g_list_first(ra_class_list); NULL!=node; node=g_list_next(node)){ + class = (char*)node->data; + if (0 == strncmp(class, rsc->class, MAX_CLASSNAMELEN)) { + ra_type_exist = TRUE; + break; + } + } + if (!ra_type_exist) { + lrmd_log(LOG_ERR + , "on_msg_add_rsc: RA class [%s] does not exist." + , rsc->class); + lrmd_rsc_destroy(rsc); + rsc = NULL; + LRMAUDIT(); + return HA_FAIL; + } + + rsc->last_op_done = NULL; + rsc->params = ha_msg_value_str_table(msg,F_LRM_PARAM); + rsc->last_op_table = g_hash_table_new(g_str_hash, g_str_equal); + g_hash_table_insert(resources, strdup(rsc->id), rsc); + + LRMAUDIT(); + return HA_OK; +} + +static int +cancel_op(GList** listp,int cancel_op_id) +{ + GList* node = NULL; + lrmd_op_t* op = NULL; + int rc = HA_FAIL; + + for( node = g_list_first(*listp) + ; node; node = g_list_next(node) ) { + op = (lrmd_op_t*)node->data; + if( op->call_id == cancel_op_id ) { + lrmd_log(LOG_INFO + ,"%s: %s cancelled" + , __FUNCTION__, op_info(op)); + rc = flush_op(op); + if( rc != HA_RSCBUSY && rc != HA_FAIL ) { + notify_client(op); /* send notification now */ + *listp = g_list_remove(*listp, op); + remove_op_history(op); + lrmd_op_destroy(op); + } + return rc; + } + } + return rc; +} + +int +on_msg_cancel_op(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc = NULL; + int cancel_op_id = 0; + int op_cancelled = HA_OK; + + LRMAUDIT(); + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_log(LOG_ERR, + "%s: no resource with such id.", __FUNCTION__); + return HA_FAIL; + } + + return_on_no_int_value(msg, F_LRM_CALLID, &cancel_op_id); + + lrmd_debug2(LOG_DEBUG + , "%s:client [pid:%d] cancel the operation [callid:%d]" + , __FUNCTION__ + , client->pid + , cancel_op_id); + + if( cancel_op(&(rsc->repeat_op_list), cancel_op_id) != HA_OK ) { + op_cancelled = cancel_op(&(rsc->op_list), cancel_op_id); + } + if( op_cancelled == HA_FAIL ) { + lrmd_log(LOG_INFO, "%s: no operation with id %d", + __FUNCTION__, cancel_op_id); + } else if( op_cancelled == HA_RSCBUSY ) { + lrmd_log(LOG_INFO, "%s: operation %d running, cancel pending", + __FUNCTION__, cancel_op_id); + } else { + lrmd_debug(LOG_DEBUG, "%s: operation %d cancelled", + __FUNCTION__, cancel_op_id); + } + LRMAUDIT(); + return op_cancelled; +} + +static gboolean +flush_all(GList** listp, int client_pid) +{ + GList* node = NULL; + lrmd_op_t* op = NULL; + gboolean rsc_busy = FALSE; + + node = g_list_first(*listp); + while( node ) { + op = (lrmd_op_t*)node->data; + if (client_pid && op->client_id != client_pid) { + node = g_list_next(node); + continue; /* not the client's operation */ + } + if( flush_op(op) == HA_RSCBUSY ) { + rsc_busy = TRUE; + node = g_list_next(node); + } else if (!client_pid || op->client_id == client_pid) { + node = *listp = g_list_remove(*listp, op); + remove_op_history(op); + lrmd_op_destroy(op); + } else { + node = g_list_next(node); + } + } + return rsc_busy; +} + +int +on_msg_flush_all(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc = NULL; + const char* id = NULL; + + LRMAUDIT(); + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + return_on_no_value(msg, F_LRM_RID,id); + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_log(LOG_ERR, + "%s: no resource with id %s.", __FUNCTION__,id); + LRMAUDIT(); + return -1; + } + + /* when a flush request arrived, flush all pending ops */ + lrmd_debug2(LOG_DEBUG + , "%s:client [%d] flush operations" + , __FUNCTION__, client->pid); + (void)flush_all(&(rsc->repeat_op_list),0); + if( flush_all(&(rsc->op_list),0) ) { + set_rsc_flushing_ops(rsc); /* resource busy */ + lrmd_log(LOG_INFO, "resource %s busy, all flush pending", rsc->id); + LRMAUDIT(); + return HA_RSCBUSY; + } + LRMAUDIT(); + return HA_OK; +} + +int +on_msg_perform_op(lrmd_client_t* client, struct ha_msg* msg) +{ + lrmd_rsc_t* rsc = NULL; + lrmd_op_t* op; + const char* id = NULL; + int timeout = 0; + int interval = 0; + int delay = 0; + + LRMAUDIT(); + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + return_on_no_value(msg, F_LRM_RID,id); + return_on_no_int_value(msg, F_LRM_INTERVAL, &interval); + return_on_no_int_value(msg, F_LRM_TIMEOUT, &timeout); + return_on_no_int_value(msg, F_LRM_DELAY, &delay); + + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_log(LOG_ERR, + "%s: no resource with such id.", __FUNCTION__); + return -1; + } + if( rsc_frozen(rsc) ) { + lrmd_log(LOG_NOTICE, "%s: resource %s is frozen, " + "no ops can run.", __FUNCTION__, rsc->id); + return -1; + } + + call_id++; + if( !(rsc->id) ) { + lrmd_debug(LOG_ERR + , "%s:%d: the resource id is NULL" + , __FUNCTION__, __LINE__); + return -1; + } + if (HA_OK != ha_msg_add_int(msg, F_LRM_CALLID, call_id)) { + LOG_FAILED_TO_ADD_FIELD("callid"); + return -1; + } + if (HA_OK !=ha_msg_mod(msg, F_LRM_APP, client->app_name)) { + LOG_FAILED_TO_ADD_FIELD("app_name"); + return -1; + } + + op = lrmd_op_new(); + if (op == NULL) { + return -1; + } + op->call_id = call_id; + op->client_id = client->pid; + op->rsc_id = strdup(rsc->id); + op->interval = interval; + op->delay = delay; + op->weight = no_child_count(rsc) ? 0 : 1; + + op->msg = ha_msg_copy(msg); + + if( ha_msg_value_int(msg,F_LRM_COPYPARAMS,&op->copyparams) == HA_OK + && op->copyparams ) { + lrmd_debug(LOG_DEBUG + , "%s:%d: copying parameters for rsc %s" + , __FUNCTION__, __LINE__,rsc->id); + if (rsc->params) { + free_str_table(rsc->params); + } + rsc->params = ha_msg_value_str_table(msg, F_LRM_PARAM); + } + + lrmd_debug2(LOG_DEBUG + , "%s: client [%d] want to add an operation %s on resource %s." + , __FUNCTION__ + , client->pid + , op_info(op) + , NULL!=op->rsc_id ? op->rsc_id : "#EMPTY#"); + + if ( 0 < op->delay ) { + op->repeat_timeout_tag = Gmain_timeout_add(op->delay + ,on_repeat_op_readytorun, op); + rsc->repeat_op_list = + g_list_append (rsc->repeat_op_list, op); + lrmd_debug(LOG_DEBUG + , "%s: an operation %s is added to the repeat " + "operation list for delay execution" + , __FUNCTION__ + , op_info(op)); + } else { + lrmd_debug(LOG_DEBUG + , "%s: add an operation %s to the operation list." + , __FUNCTION__ + , op_info(op)); + add_op_to_runlist(rsc,op); + } + + perform_op(rsc); + + LRMAUDIT(); + return call_id; +} + +static void +send_last_op(gpointer key, gpointer value, gpointer user_data) +{ + IPC_Channel* ch = NULL; + lrmd_op_t* op = NULL; + struct ha_msg* msg = NULL; + + ch = (IPC_Channel*)user_data; + op = (lrmd_op_t*)value; + msg = op_to_msg(op); + if (msg == NULL) { + lrmd_log(LOG_ERR, "send_last_op: failed to convert an operation " + "information to a ha_msg."); + return; + } + if (HA_OK != msg2ipcchan(msg, ch)) { + lrmd_log(LOG_ERR, "send_last_op: can not send a message."); + } + ha_msg_del(msg); +} + +int +on_msg_get_state(lrmd_client_t* client, struct ha_msg* msg) +{ + int op_count = 0; + lrmd_rsc_t* rsc = NULL; + GList* node; + struct ha_msg* ret = NULL; + lrmd_op_t* op = NULL; + struct ha_msg* op_msg = NULL; + const char* id = NULL; + GHashTable* last_ops = NULL; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + id = ha_msg_value(msg,F_LRM_RID); + lrmd_debug2(LOG_DEBUG + , "%s: client [%d] want to get the state of resource %s" + , __FUNCTION__, client->pid, lrmd_nullcheck(id)); + + rsc = lookup_rsc_by_msg(msg); + if (NULL == rsc) { + lrmd_log(LOG_ERR, "on_msg_get_state: no resource with id %s." + , lrmd_nullcheck(id)); + send_ret_msg(client->ch_cmd, HA_FAIL); + return HA_FAIL; + } + + ret = ha_msg_new(5); + if (NULL == ret) { + lrmd_log(LOG_ERR, "on_msg_get_state: can't create a ha_msg."); + return HA_FAIL; + } + /* add the F_LRM_STATE field */ + if (HA_OK != ha_msg_add_int(ret, F_LRM_STATE + , rsc->op_list ? LRM_RSC_BUSY : LRM_RSC_IDLE)) { + LOG_FAILED_TO_ADD_FIELD("state"); + ha_msg_del(ret); + return HA_FAIL; + } + lrmd_debug(LOG_DEBUG + , "on_msg_get_state:state of rsc %s is %s" + , lrmd_nullcheck(id) + , rsc->op_list ? "LRM_RSC_BUSY" : "LRM_RSC_IDLE" ); + /* calculate the count of ops being returned */ + last_ops = g_hash_table_lookup(rsc->last_op_table, client->app_name); + if (last_ops == NULL) { + op_count = g_list_length(rsc->op_list) + + g_list_length(rsc->repeat_op_list); + } + else { + op_count = g_hash_table_size(last_ops) + + g_list_length(rsc->op_list) + + g_list_length(rsc->repeat_op_list); + } + /* add the count of ops being returned */ + if (HA_OK != ha_msg_add_int(ret, F_LRM_OPCNT, op_count)) { + LOG_FAILED_TO_ADD_FIELD("operation count"); + ha_msg_del(ret); + return HA_FAIL; + } + /* send the first message to client */ + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_state: can not send the ret message."); + ha_msg_del(ret); + return HA_FAIL; + } + ha_msg_del(ret); + + /* send the ops in last ops table */ + if(last_ops != NULL) { + g_hash_table_foreach(last_ops, send_last_op, client->ch_cmd); + } + + /* send the ops in op list */ + for(node = g_list_first(rsc->op_list) + ; NULL != node; node = g_list_next(node)){ + op = (lrmd_op_t*)node->data; + op_msg = op_to_msg(op); + if (NULL == op_msg) { + lrmd_log(LOG_ERR, + "on_msg_get_state: failed to make a message " + "from a operation: %s", op_info(op)); + continue; + } + if (HA_OK != msg2ipcchan(op_msg, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_state: failed to send a message."); + } + ha_msg_del(op_msg); + } + + /* send the ops in repeat op list */ + for(node = g_list_first(rsc->repeat_op_list) + ; NULL != node; node = g_list_next(node)){ + op = (lrmd_op_t*)node->data; + op_msg = op_to_msg(op); + if (NULL == op_msg) { + lrmd_log(LOG_ERR, + "on_msg_get_state: failed to make a message " + "from a operation: %s", op_info(op)); + continue; + } + if (HA_OK != msg2ipcchan(op_msg, client->ch_cmd)) { + lrmd_log(LOG_ERR, + "on_msg_get_state: failed to send a message."); + } + ha_msg_del(op_msg); + } + return HA_OK; +} + +#define safe_len(s) (s ? strlen(s) : 0) + +static char * +lrm_concat(const char *prefix, const char *suffix, char join) +{ + int len = 2; + char *new_str = NULL; + len += safe_len(prefix); + len += safe_len(suffix); + + new_str = malloc(sizeof(char)*len); + if (NULL == new_str) { + lrmd_log(LOG_ERR,"%s:%d: malloc failed" + , __FUNCTION__, __LINE__); + return NULL; + } + + memset(new_str, 0, len); + sprintf(new_str, "%s%c%s", prefix?prefix:"", join, suffix?suffix:""); + new_str[len-1] = 0; + return new_str; +} + +/* /////////////////////op functions////////////////////// */ + +#define mk_op_id(op,id) do { \ + const char *op_type = ha_msg_value(op->msg, F_LRM_OP); \ + const char *op_interval = ha_msg_value(op->msg, F_LRM_INTERVAL); \ + id = lrm_concat(op_type, op_interval, '_'); \ +} while(0) + +/* find the last operation for the client + * replace it with the new one (if requested) + */ +static void +replace_last_op(lrmd_client_t* client, lrmd_rsc_t* rsc, lrmd_op_t* op) +{ + char *op_hash_key; + GHashTable *client_last_op; + lrmd_op_t *old_op, *new_op; + + if (!client || !rsc || !op) + return; + client_last_op = g_hash_table_lookup(rsc->last_op_table, client->app_name); + if (!client_last_op) { + lrmd_debug2(LOG_DEBUG + , "%s: new last op table for client %s" + , __FUNCTION__, client->app_name); + client_last_op = g_hash_table_new_full( g_str_hash + , g_str_equal, free, NULL); + g_hash_table_insert(rsc->last_op_table + , (gpointer)strdup(client->app_name) + , (gpointer)client_last_op); + } + mk_op_id(op,op_hash_key); + old_op = (lrmd_op_t*)g_hash_table_lookup(client_last_op, op_hash_key); + + /* make a copy of op and insert it into client_last_op */ + if (!(new_op = lrmd_op_copy(op))) { + lrmd_log(LOG_ERR, "%s:%d out of memory" + , __FUNCTION__, __LINE__); + } + if (old_op) { + lrmd_debug2(LOG_DEBUG + , "%s: replace last op %s for client %s" + , __FUNCTION__, op_hash_key, client->app_name); + g_hash_table_replace(client_last_op,op_hash_key,(gpointer)new_op); + lrmd_op_destroy(old_op); + } else { + lrmd_debug2(LOG_DEBUG + , "%s: add last op %s for client %s" + , __FUNCTION__, op_hash_key, client->app_name); + g_hash_table_insert(client_last_op,op_hash_key,(gpointer)new_op); + } +} + +static int +record_op_completion(lrmd_rsc_t* rsc, lrmd_op_t* op) +{ + lrmd_client_t* client; + + LRMAUDIT(); + /*save the op in the last op finished*/ + if (rsc->last_op_done != NULL) { + lrmd_op_destroy(rsc->last_op_done); + } + if (!(rsc->last_op_done = lrmd_op_copy(op))) { + lrmd_log(LOG_ERR, "%s:%d out of memory" + , __FUNCTION__, __LINE__); + return 1; + } + rsc->last_op_done->repeat_timeout_tag = (guint)0; + + client = lookup_client(op->client_id); + if (!client) { + lrmd_log(LOG_INFO, "%s: cannot record %s: the client is gone" + , __FUNCTION__, small_op_info(op)); + LRMAUDIT(); + return 1; + } + /* insert (or replace) the new op in last_op_table for the client */ + replace_last_op(client,rsc,op); + LRMAUDIT(); + return 0; +} + +static void +to_repeatlist(lrmd_rsc_t* rsc, lrmd_op_t* op) +{ + lrmd_op_t *repeat_op; + + if (!(repeat_op = lrmd_op_copy(op))) { + lrmd_log(LOG_ERR, "%s:%d out of memory" + , __FUNCTION__, __LINE__); + } + reset_timestamps(repeat_op); + repeat_op->is_copy = FALSE; + repeat_op->repeat_timeout_tag = + Gmain_timeout_add(op->interval, + on_repeat_op_readytorun, repeat_op); + rsc->repeat_op_list = + g_list_append (rsc->repeat_op_list, repeat_op); + lrmd_debug2(LOG_DEBUG + , "%s: repeat %s is added to repeat op list to wait" + , __FUNCTION__, op_info(op)); +} + +static void +remove_op_history(lrmd_op_t* op) +{ + lrmd_client_t* client = lookup_client(op->client_id); + lrmd_rsc_t* rsc = NULL; + char *op_id, *last_op_id; + lrmd_op_t* old_op = NULL; + GHashTable* client_last_op = NULL; + + LRMAUDIT(); + if( !(rsc = lookup_rsc(op->rsc_id)) ) { + return; + } + lrmd_debug2(LOG_DEBUG, "%s: remove history of the op %s" + ,__FUNCTION__, op_info(op)); + mk_op_id(op,op_id); + if (rsc->last_op_done != NULL ) { + mk_op_id(rsc->last_op_done,last_op_id); + if( !strcmp(op_id,last_op_id) ) { + lrmd_debug2(LOG_DEBUG, "%s: remove history of the last op done %s" + ,__FUNCTION__, op_info(rsc->last_op_done)); + lrmd_op_destroy(rsc->last_op_done); + rsc->last_op_done = NULL; + } + free(last_op_id); + } + if( client && + (client_last_op = g_hash_table_lookup(rsc->last_op_table + , client->app_name)) ) { + lrmd_debug2(LOG_DEBUG, "%s: found client %s in the last op table" + ,__FUNCTION__, client->app_name); + old_op = g_hash_table_lookup(client_last_op, op_id); + if (old_op) { + g_hash_table_remove(client_last_op, op_id); + lrmd_debug2(LOG_DEBUG, "%s: remove history of the client's last %s" + ,__FUNCTION__, op_info(old_op)); + lrmd_op_destroy(old_op); + } + } + free(op_id); + LRMAUDIT(); +} + +static void +add_op_to_runlist(lrmd_rsc_t* rsc, lrmd_op_t* op) +{ + op->t_addtolist = time_longclock(); + rsc->op_list = g_list_append(rsc->op_list, op); + if (g_list_length(rsc->op_list) >= 4) { + lrmd_log(LOG_WARNING + , "operations list for %s is suspiciously" + " long [%d]" + , rsc->id + , g_list_length(rsc->op_list)); + lrmd_rsc_dump(rsc->id, "rsc->op_list: too many ops"); + } +} + +/* 1. this function sends a message to the client: + * a) on operation instance exit using the callback channel + * b) in case a client requested that operation to be cancelled, + * using the command channel + * c) in case a client requested a resource removal or flushing + * all ops and this is the last operation that finished, again + * using the command channel + * 2. if the op was not cancelled: + * a) it is copied to the last_op_done field of rsc + * b) if it's a repeating op, it is put in the repeat_op_list + * c) the outcome is recorded for future reference + * 3. op is destroyed and removed from the op_list + */ +int +on_op_done(lrmd_rsc_t* rsc, lrmd_op_t* op) +{ + int rc = HA_OK; + int target_rc, last_rc, op_rc; + int rc_changed; + op_status_t op_status; + + LRMAUDIT(); + CHECK_ALLOCATED(op, "op", HA_FAIL ); + if (op->exec_pid == 0) { + lrmd_log(LOG_ERR, "%s: op->exec_pid == 0",__FUNCTION__); + return HA_FAIL; + } + op->t_done = time_longclock(); + + if (debug_level >= 2) { + lrmd_debug(LOG_DEBUG, "%s: %s",__FUNCTION__, op_info(op)); + lrmd_op_dump(op, __FUNCTION__); + } + + return_on_no_int_value(op->msg,F_LRM_TARGETRC,&target_rc); + return_on_no_int_value(op->msg,F_LRM_OPSTATUS,(int *)&op_status); + + last_rc = op_rc = -1; /* set all rc to -1 */ + ha_msg_value_int(op->msg,F_LRM_RC,&op_rc); + ha_msg_value_int(op->msg,F_LRM_LASTRC,&last_rc); + rc_changed = ( + op_status == LRM_OP_DONE + && op_rc != -1 + && ((last_rc == -1) || (last_rc != op_rc)) + ); + if (rc_changed) { + if (HA_OK != ha_msg_mod_int(op->msg, F_LRM_LASTRC, op_rc)) { + lrmd_log(LOG_ERR,"%s: cannot save status to msg",__FUNCTION__); + return HA_FAIL; + } + op->t_rcchange = op->t_perform; + } + if (store_timestamps(op)) + return HA_FAIL; + + /* remove the op from op_list */ + rsc->op_list = g_list_remove(rsc->op_list,op); + lrmd_debug2(LOG_DEBUG + , "%s:%s is removed from op list" + , __FUNCTION__, op_info(op)); + + if (!op->is_cancelled) { + if( !record_op_completion(rsc,op) ) { /*record the outcome of the op */ + if (op->interval) /* copy op to the repeat list */ + to_repeatlist(rsc,op); + } + } else { + if (HA_OK != ha_msg_mod_int(op->msg,F_LRM_OPSTATUS,(int)LRM_OP_CANCELLED)) { + LOG_FAILED_TO_ADD_FIELD(F_LRM_OPSTATUS); + return HA_FAIL; + } + op_status = LRM_OP_CANCELLED; + remove_op_history(op); + } + + if (rsc_removal_pending(rsc)) { + if (HA_OK != ha_msg_add_int(op->msg,F_LRM_RSCDELETED,1)) { + LOG_FAILED_TO_ADD_FIELD(F_LRM_RSCDELETED); + } + } + if (op_status != LRM_OP_DONE + || (op_rc == -1) + || (op_rc == target_rc) + || (target_rc == EVERYTIME) + || ((target_rc == CHANGED) && rc_changed) + || rsc_removal_pending(rsc) + ) { + notify_client(op); + } + lrmd_op_destroy(op); + if( !rsc->op_list ) { + if( rsc_removal_pending(rsc) ) { + lrmd_log(LOG_INFO, "late removal of resource %s", rsc->id); + lrmd_rsc_destroy(rsc); + rc = -1; /* let the caller know that the rsc is gone */ + } else { + rsc_reset_state(rsc); + } + } + LRMAUDIT(); + if (shutdown_in_progress && can_shutdown()) { + lrm_shutdown(); + } + return rc; +} + +/* + * an operation is flushed only in case there is + * no process running initiated by this operation + * NB: the caller has to destroy the operation itself + */ +int +flush_op(lrmd_op_t* op) +{ + CHECK_ALLOCATED(op, "op", HA_FAIL ); + if (op->exec_pid == 0) { + lrmd_debug(LOG_ERR, "%s: op->exec_pid == 0",__FUNCTION__); + return HA_FAIL; + } + + if (HA_OK != ha_msg_mod_int(op->msg, F_LRM_RC, HA_FAIL)) { + LOG_FAILED_TO_ADD_FIELD("F_LRM_RC"); + return HA_FAIL; + } + + if( op->exec_pid == -1 ) { + if (HA_OK != ha_msg_mod_int(op->msg,F_LRM_OPSTATUS,(int)LRM_OP_CANCELLED)){ + LOG_FAILED_TO_ADD_FIELD("opstatus"); + return HA_FAIL; + } + return HA_OK; + } else { + op->is_cancelled = TRUE; /* mark the op as cancelled */ + lrmd_log(LOG_INFO, "%s: process for %s still " + "running, flush delayed" + ,__FUNCTION__,small_op_info(op)); + return HA_RSCBUSY; + } +} + +/* Resume the execution of ops of the resource */ +static gboolean +rsc_execution_freeze_timeout(gpointer data) +{ + lrmd_rsc_t* rsc = (lrmd_rsc_t*)data; + + if (rsc == NULL) { + return FALSE; + } + + if (rsc->delay_timeout > 0) { + rsc->delay_timeout = (guint)0; + } + + perform_op(rsc); + + return FALSE; +} + +/* this function gets the first op in the rsc op list and execute it*/ +int +perform_op(lrmd_rsc_t* rsc) +{ + GList* node = NULL; + lrmd_op_t* op = NULL; + + LRMAUDIT(); + CHECK_ALLOCATED(rsc, "resource", HA_FAIL); + if (shutdown_in_progress && can_shutdown()) { + lrm_shutdown(); + } + + if (rsc_frozen(rsc)) { + lrmd_log(LOG_INFO,"%s: resource %s is frozen, " + "no ops allowed to run" + , __FUNCTION__, rsc->id); + return HA_OK; + } + + if (NULL == rsc->op_list) { + lrmd_debug2(LOG_DEBUG,"%s: no op to perform?", __FUNCTION__); + return HA_OK; + } + + node = g_list_first(rsc->op_list); + while (NULL != node) { + op = node->data; + if (-1 != op->exec_pid) { + if (!g_list_next(node)) { + /* this is the only operation, no need to do + * anything further */ + break; + } + lrmd_log(LOG_INFO, "%s:%d: %s for rsc is already running." + , __FUNCTION__, __LINE__, op_info(op)); + if( rsc->delay_timeout > 0 ) { + lrmd_log(LOG_INFO + , "%s:%d: operations on resource %s already delayed" + , __FUNCTION__, __LINE__, lrm_str(rsc->id)); + } else { + lrmd_log(LOG_INFO + , "%s:%d: postponing " + "all ops on resource %s by %d ms" + , __FUNCTION__, __LINE__ + , lrm_str(rsc->id), retry_interval); + rsc->delay_timeout = Gmain_timeout_add(retry_interval + , rsc_execution_freeze_timeout, rsc); + } + break; + } + if (op->weight && child_count >= max_child_count) { + if ((int)rsc->delay_timeout > 0) { + lrmd_log(LOG_INFO + , "%s:%d: max_child_count (%d) reached and operations on resource %s already delayed" + , __FUNCTION__, __LINE__, max_child_count, lrm_str(rsc->id)); + } else { + lrmd_debug(LOG_NOTICE + , "max_child_count (%d) reached, postponing " + "execution of %s by %d ms" + , max_child_count, op_info(op), retry_interval); + rsc->delay_timeout = Gmain_timeout_add(retry_interval + , rsc_execution_freeze_timeout, rsc); + } + break; + } + + if (HA_OK != perform_ra_op(op)) { + lrmd_log(LOG_ERR + , "unable to perform_ra_op on %s" + , op_info(op)); + if (HA_OK != ha_msg_add_int(op->msg, F_LRM_OPSTATUS, + LRM_OP_ERROR)) { + LOG_FAILED_TO_ADD_FIELD("opstatus"); + } + on_op_done(rsc,op); + node = g_list_first(rsc->op_list); + } + else { + break; + } + } + + LRMAUDIT(); + return HA_OK; +} + +static int +store_timestamps(lrmd_op_t* op) +{ + struct ha_msg* msg = op->msg; + longclock_t now = time_longclock(), /* tm2unix() needs this */ + exec_time = zero_longclock, + queue_time = zero_longclock; + + if (op->t_perform) { + queue_time = + longclockto_ms(sub_longclock(op->t_perform,op->t_addtolist)); + if (op->t_done) { + exec_time = + longclockto_ms(sub_longclock(op->t_done,op->t_perform)); + } + } + if ((HA_OK!=ha_msg_mod_ul(msg,F_LRM_T_RUN,tm2unix(op->t_perform))) + || (HA_OK!=ha_msg_mod_ul(msg,F_LRM_T_RCCHANGE,tm2unix(op->t_rcchange))) + || (HA_OK!=ha_msg_mod_ul(msg,F_LRM_EXEC_TIME,exec_time)) + || (HA_OK!=ha_msg_mod_ul(msg,F_LRM_QUEUE_TIME,queue_time)) + ) { + lrmd_log(LOG_ERR,"%s: can not save timestamps to msg",__FUNCTION__); + return 1; + } + return 0; +} + +static void +reset_timestamps(lrmd_op_t* op) +{ + op->t_perform = zero_longclock; + op->t_done = zero_longclock; + cl_msg_remove(op->msg, F_LRM_T_RUN); + cl_msg_remove(op->msg, F_LRM_T_RCCHANGE); + cl_msg_remove(op->msg, F_LRM_EXEC_TIME); + cl_msg_remove(op->msg, F_LRM_QUEUE_TIME); +} + +struct ha_msg* +op_to_msg(lrmd_op_t* op) +{ + struct ha_msg* msg = NULL; + + CHECK_ALLOCATED(op, "op", NULL); + if (op->exec_pid == 0) { + lrmd_log(LOG_ERR, "%s: op->exec_pid is 0",__FUNCTION__); + return NULL; + } + msg = ha_msg_copy(op->msg); + if (NULL == msg) { + lrmd_log(LOG_ERR,"%s: can not copy the msg",__FUNCTION__); + return NULL; + } + if ((HA_OK!=ha_msg_mod_int(msg,F_LRM_CALLID,op->call_id))) { + lrmd_log(LOG_ERR,"%s: can not save F_LRM_CALLID to msg",__FUNCTION__); + ha_msg_del(msg); + msg = NULL; + } + return msg; +} + +/* //////////////////////////////RA wrap funcs/////////////////////////////////// */ +int +perform_ra_op(lrmd_op_t* op) +{ + int stdout_fd[2]; + int stderr_fd[2]; + pid_t pid; + int timeout; + struct RAExecOps * RAExec = NULL; + const char* op_type = NULL; + GHashTable* params = NULL; + GHashTable* op_params = NULL; + lrmd_rsc_t* rsc = NULL; + ra_pipe_op_t * rapop; + + LRMAUDIT(); + CHECK_ALLOCATED(op, "op", HA_FAIL); + rsc = (lrmd_rsc_t*)lookup_rsc(op->rsc_id); + CHECK_ALLOCATED(rsc, "rsc", HA_FAIL); + + if ( pipe(stdout_fd) < 0 ) { + cl_perror("%s::%d: pipe", __FUNCTION__, __LINE__); + } + + if ( pipe(stderr_fd) < 0 ) { + cl_perror("%s::%d: pipe", __FUNCTION__, __LINE__); + } + + if (op->exec_pid == 0) { + lrmd_log(LOG_ERR, "%s::%d: op->exec_pid == 0.", __FUNCTION__, __LINE__); + return HA_FAIL; + } + + op_type = ha_msg_value(op->msg, F_LRM_OP); + op->t_perform = time_longclock(); + check_queue_duration(op); + + if(HA_OK != ha_msg_value_int(op->msg, F_LRM_TIMEOUT, &timeout)){ + timeout = 0; + lrmd_log(LOG_ERR,"%s::%d: failed to get timeout for %s" + , __FUNCTION__, __LINE__, small_op_info(op)); + } + + if( return_to_orig_privs() ) { + cl_perror("%s::%d: failed to raise privileges" + , __FUNCTION__, __LINE__); + } + switch(pid=fork()) { + case -1: + cl_perror("%s::%d: fork", __FUNCTION__, __LINE__); + close(stdout_fd[0]); + close(stdout_fd[1]); + close(stderr_fd[0]); + close(stderr_fd[1]); + if( return_to_dropped_privs() ) { + cl_perror("%s::%d: failed to drop privileges" + , __FUNCTION__, __LINE__); + } + return HA_FAIL; + + default: /* Parent */ + child_count += op->weight; + NewTrackedProc(pid, 1 + , debug_level ? + ((op->interval && !is_logmsg_due(op)) ? PT_LOGNORMAL : PT_LOGVERBOSE) : PT_LOGNONE + , op, &ManagedChildTrackOps); + + if (!op->interval || is_logmsg_due(op)) { /* log non-repeating ops */ + lrmd_log(LOG_INFO,"rsc:%s %s[%d] (pid %d)", + rsc->id,probe_str(op,op_type),op->call_id,pid); + } else { + lrmd_debug(LOG_DEBUG,"rsc:%s %s[%d] (pid %d)", + rsc->id,op_type,op->call_id,pid); + } + close(stdout_fd[1]); + close(stderr_fd[1]); + rapop = ra_pipe_op_new(stdout_fd[0], stderr_fd[0], op); + op->rapop = rapop; + op->exec_pid = pid; + if (0 < timeout ) { + + /* Wait 'timeout' ms then send SIGTERM */ + /* allow for extra 15 seconds for stonith, + * because stonithd handles its children with the + * same timeout; in this case the lrmd child + * should never timeout, but return the timeout + * reported by stonithd + */ + op->killseq[0].mstimeout = timeout + + (!strcmp(rsc->class,"stonith") ? 15000 : 0); + op->killseq[0].signalno = SIGTERM; + + /* Wait 5 seconds then send SIGKILL */ + op->killseq[1].mstimeout = 5000; + op->killseq[1].signalno = SIGKILL; + + /* Wait 5 more seconds then moan and complain */ + op->killseq[2].mstimeout = 5000; + op->killseq[2].signalno = 0; + + SetTrackedProcTimeouts(pid, op->killseq); + } + if( return_to_dropped_privs() ) { + lrmd_log(LOG_WARNING,"%s::%d: failed to drop privileges: %s" + , __FUNCTION__, __LINE__, strerror(errno)); + } + + if ( rapop == NULL) { + return HA_FAIL; + } + LRMAUDIT(); + return HA_OK; + + case 0: /* Child */ +#ifdef DEFAULT_REALTIME_POLICY + if (sched_getscheduler(0) != SCHED_OTHER) { + struct sched_param sp; + lrmd_debug(LOG_DEBUG, + "perform_ra_op: resetting scheduler class to SCHED_OTHER"); + sp.sched_priority = 0; + if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) + cl_perror("%s::%d: sched_setscheduler", + __FUNCTION__, __LINE__); + } +#endif + /* Man: The call setpgrp() is equivalent to setpgid(0,0) + * _and_ compiles on BSD variants too + * need to investigate if it works the same too. + */ + setpgid(0,0); + close(stdout_fd[0]); + close(stderr_fd[0]); + if (STDOUT_FILENO != stdout_fd[1]) { + if (dup2(stdout_fd[1], STDOUT_FILENO)!=STDOUT_FILENO) { + cl_perror("%s::%d: dup2" + , __FUNCTION__, __LINE__); + } + close(stdout_fd[1]); + } + if (STDERR_FILENO != stderr_fd[1]) { + if (dup2(stderr_fd[1], STDERR_FILENO)!=STDERR_FILENO) { + cl_perror("%s::%d: dup2", __FUNCTION__, __LINE__); + } + close(stderr_fd[1]); + } + RAExec = g_hash_table_lookup(RAExecFuncs,rsc->class); + if (NULL == RAExec) { + close(stdout_fd[1]); + close(stderr_fd[1]); + lrmd_log(LOG_ERR,"%s::%d: can't find RAExec for class %s" + , __FUNCTION__, __LINE__, rsc->class); + exit(EXECRA_EXEC_UNKNOWN_ERROR); + } + + /*should we use logging daemon or not in script*/ + setenv(HALOGD, cl_log_get_uselogd()?"yes":"no",1); + + /* Name of the resource and some others also + * need to be passed in. Maybe pass through the + * entire lrm_op_t too? */ + lrmd_debug2(LOG_DEBUG + , "perform_ra_op:calling RA plugin to perform %s, pid: [%d]" + , op_info(op), getpid()); + + op_params = ha_msg_value_str_table(op->msg, F_LRM_PARAM); + params = merge_str_tables(rsc->params,op_params); + if (op_params) { + free_str_table(op_params); + op_params = NULL; + } + + if (replace_secret_params(rsc->id, params) < 0) { + /* replacing secrets failed! */ + if (!strcmp(op_type,"stop")) { + /* don't fail on stop! */ + lrmd_log(LOG_INFO + , "%s:%d: proceeding with the stop operation for %s" + , __FUNCTION__, __LINE__, rsc->id); + } else { + lrmd_log(LOG_ERR + , "%s:%d: failed to get secrets for %s, " + "considering resource not configured" + , __FUNCTION__, __LINE__, rsc->id); + exit(EXECRA_NOT_CONFIGURED); + } + } + RAExec->execra (rsc->id, + rsc->type, + rsc->provider, + op_type, + timeout, + params); + + /* execra should never return. */ + exit(EXECRA_EXEC_UNKNOWN_ERROR); + + } + lrmd_log(LOG_ERR, "perform_ra_op: end(impossible)."); + return HA_OK; +} + +static void +on_ra_proc_registered(ProcTrack* p) +{ +} + +/* Handle one of our ra child processes finished*/ +static void +on_ra_proc_finished(ProcTrack* p, int status, int signo, int exitcode +, int waslogged) +{ + lrmd_op_t* op = NULL; + lrmd_rsc_t* rsc = NULL; + struct RAExecOps * RAExec = NULL; + const char* op_type; + int rc = EXECRA_EXEC_UNKNOWN_ERROR; + int ret; + int op_status; + + LRMAUDIT(); + + CHECK_ALLOCATED(p, "ProcTrack p", ); + op = proctrack_data(p); + + child_count -= op->weight; + if (child_count < 0) { + lrmd_log(LOG_ERR, "%s:%d: child count is less than zero: %d" + , __FUNCTION__, __LINE__, child_count); + child_count = 0; + } + + lrmd_debug2(LOG_DEBUG, "on_ra_proc_finished: accessing the op whose " + "address is %p", op); + CHECK_ALLOCATED(op, "op", ); + if (op->exec_pid == 0) { + lrmd_log(LOG_ERR, "on_ra_proc_finished: the op was freed."); + dump_data_for_debug(); + return; + } + RemoveTrackedProcTimeouts(op->exec_pid); + op->exec_pid = -1; + + rsc = lookup_rsc(op->rsc_id); + if (rsc == NULL) { + lrmd_log(LOG_ERR, "%s: the rsc (id=%s) does not exist" + , __FUNCTION__, lrm_str(op->rsc_id)); + lrmd_op_dump(op, __FUNCTION__); + lrmd_dump_all_resources(); + /* delete the op */ + lrmd_op_destroy(op); + reset_proctrack_data(p); + LRMAUDIT(); + return; + } + + RAExec = g_hash_table_lookup(RAExecFuncs,rsc->class); + if (NULL == RAExec) { + lrmd_log(LOG_ERR,"on_ra_proc_finished: can not find RAExec for" + " resource class <%s>", rsc->class); + dump_data_for_debug(); + return; + } + + op_type = ha_msg_value(op->msg, F_LRM_OP); + + if ( (NULL == strchr(op->first_line_ra_stdout, '\n')) + && (0==STRNCMP_CONST(rsc->class, "heartbeat")) + && ( (0==STRNCMP_CONST(op_type, "monitor")) + ||(0==STRNCMP_CONST(op_type, "status"))) ) { + if ( ( op->rapop != NULL ) + && (op->rapop->ra_stdout_fd >= 0) ) { + handle_pipe_ra_stdout(op->rapop->ra_stdout_fd + , op->rapop); + } else { + lrmd_log(LOG_WARNING, "There is something wrong: the " + "first line isn't read in. Maybe the heartbeat " + "does not ouput string correctly for status " + "operation. Or the code (myself) is wrong."); + } + } + + if( signo ) { + if( proctrack_timedout(p) ) { + lrmd_log(LOG_WARNING, "%s: pid %d timed out" + , small_op_info(op), proctrack_pid(p)); + op_status = LRM_OP_TIMEOUT; + } else { + op_status = LRM_OP_ERROR; + } + } else { + rc = RAExec->map_ra_retvalue(exitcode, op_type + , op->first_line_ra_stdout); + if (!op->interval || is_logmsg_due(op) || debug_level > 0) { /* log non-repeating ops */ + if (rc == exitcode) { + lrmd_log(LOG_INFO + , "%s: pid %d exited with" + " return code %d", small_op_info(op), proctrack_pid(p), rc); + }else{ + lrmd_log(LOG_INFO + , "%s: pid %d exited with" + " return code %d (mapped from %d)" + , small_op_info(op), proctrack_pid(p), rc, exitcode); + } + } + if (EXECRA_EXEC_UNKNOWN_ERROR == rc || EXECRA_NO_RA == rc) { + op_status = LRM_OP_ERROR; + lrmd_log(LOG_CRIT + , "on_ra_proc_finished: the exit code indicates a problem."); + } else { + op_status = LRM_OP_DONE; + } + } + if (op->interval && is_logmsg_due(op)) { + op->t_lastlogmsg = time_longclock(); + } + if (HA_OK != + ha_msg_mod_int(op->msg, F_LRM_OPSTATUS, op_status)) { + LOG_FAILED_TO_ADD_FIELD("opstatus"); + return ; + } + if (HA_OK != ha_msg_mod_int(op->msg, F_LRM_RC, rc)) { + LOG_FAILED_TO_ADD_FIELD("F_LRM_RC"); + return ; + } + + if ( 0 < strlen(op->first_line_ra_stdout) ) { + if (NULL != cl_get_string(op->msg, F_LRM_DATA)) { + cl_msg_remove(op->msg, F_LRM_DATA); + } + ret = ha_msg_add(op->msg, F_LRM_DATA, op->first_line_ra_stdout); + if (HA_OK != ret) { + LOG_FAILED_TO_ADD_FIELD("data"); + } + } + + if (on_op_done(rsc,op) >= 0) { + perform_op(rsc); + } + reset_proctrack_data(p); + LRMAUDIT(); +} + +/* Handle the death of one of our managed child processes */ +static const char * +on_ra_proc_query_name(ProcTrack* p) +{ + static char proc_name[MAX_PROC_NAME]; + lrmd_op_t* op = NULL; + lrmd_rsc_t* rsc = NULL; + const char* op_type = NULL; + + LRMAUDIT(); + op = (lrmd_op_t*)(proctrack_data(p)); + if (NULL == op || op->exec_pid == 0) { + return "*unknown*"; + } + + op_type = ha_msg_value(op->msg, F_LRM_OP); + rsc = lookup_rsc(op->rsc_id); + if (rsc == NULL) { + snprintf(proc_name + , MAX_PROC_NAME + , "unknown rsc(%s):%s maybe deleted" + , op->rsc_id, op_type); + }else { + snprintf(proc_name, MAX_PROC_NAME, "%s:%s", rsc->id, op_type); + } + LRMAUDIT(); + return proc_name; +} + +static int +get_lrmd_param(const char *name, char *value, int maxstring) +{ + if (!name) { + lrmd_log(LOG_ERR, "%s: empty name", __FUNCTION__); + return HA_FAIL; + } + if (!strcmp(name,"max-children")) { + snprintf(value, maxstring, "%d", max_child_count); + return HA_OK; + } else { + lrmd_log(LOG_ERR, "%s: unknown lrmd parameter %s", __FUNCTION__, name); + return HA_FAIL; + } +} + +static int +set_lrmd_param(const char *name, const char *value) +{ + int ival; + + if (!name) { + lrmd_log(LOG_ERR, "%s: empty name", __FUNCTION__); + return HA_FAIL; + } + if (!value) { + lrmd_log(LOG_ERR, "%s: empty value", __FUNCTION__); + return HA_FAIL; + } + if (!strcmp(name,"max-children")) { + ival = atoi(value); + if (ival <= 0) { + lrmd_log(LOG_ERR, "%s: invalid value for lrmd parameter %s" + , __FUNCTION__, name); + return HA_FAIL; + } else if (ival == max_child_count) { + lrmd_log(LOG_INFO, "max-children already set to %d", ival); + return HA_OK; + } + lrmd_log(LOG_INFO, "setting max-children to %d", ival); + max_child_count = ival; + return HA_OK; + } else { + lrmd_log(LOG_ERR, "%s: unknown lrmd parameter %s" + , __FUNCTION__, name); + return HA_FAIL; + } +} + +int +on_msg_set_lrmd_param(lrmd_client_t* client, struct ha_msg* msg) +{ + const char *name, *value; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + name = ha_msg_value(msg,F_LRM_LRMD_PARAM_NAME); + value = ha_msg_value(msg,F_LRM_LRMD_PARAM_VAL); + if (!name || !value) { + lrmd_log(LOG_ERR, "%s: no parameter defined" + , __FUNCTION__); + return HA_FAIL; + } + return set_lrmd_param(name,value); +} + +int +on_msg_get_lrmd_param(lrmd_client_t* client, struct ha_msg* msg) +{ + struct ha_msg* ret = NULL; + const char *name; + char value[MAX_NAME_LEN]; + + CHECK_ALLOCATED(client, "client", HA_FAIL); + CHECK_ALLOCATED(msg, "message", HA_FAIL); + + ret = create_lrm_ret(HA_OK, 1); + CHECK_RETURN_OF_CREATE_LRM_RET; + + name = ha_msg_value(msg,F_LRM_LRMD_PARAM_NAME); + if (get_lrmd_param(name, value, MAX_NAME_LEN) != HA_OK) { + return HA_FAIL; + } + if (HA_OK != ha_msg_add(ret, F_LRM_LRMD_PARAM_VAL, value)) { + ha_msg_del(ret); + LOG_FAILED_TO_ADD_FIELD(F_LRM_LRMD_PARAM_VAL); + return HA_FAIL; + } + if (HA_OK != msg2ipcchan(ret, client->ch_cmd)) { + lrmd_log(LOG_ERR, "%s: can not send the ret msg",__FUNCTION__); + } + ha_msg_del(ret); + return HA_OK; +} + + +/* /////////////////Util Functions////////////////////////////////////////////// */ +int +send_ret_msg (IPC_Channel* ch, int ret) +{ + struct ha_msg* msg = NULL; + + msg = create_lrm_ret(ret, 1); + CHECK_RETURN_OF_CREATE_LRM_RET; + + if (HA_OK != msg2ipcchan(msg, ch)) { + lrmd_log(LOG_ERR, "send_ret_msg: can not send the ret msg"); + } + ha_msg_del(msg); + return HA_OK; +} + +static void +send_cbk_msg(struct ha_msg* msg, lrmd_client_t* client) +{ + if (!client) { + lrmd_log(LOG_WARNING, + "%s: zero client", __FUNCTION__); + return; + } + if (!client->ch_cbk) { + lrmd_log(LOG_WARNING, + "%s: callback channel is null", __FUNCTION__); + } else if (HA_OK != msg2ipcchan(msg, client->ch_cbk)) { + lrmd_log(LOG_WARNING, + "%s: can not send the ret msg", __FUNCTION__); + } +} + +static void +send_msg(struct ha_msg* msg, lrmd_client_t* client) +{ + if (!client) { + lrmd_log(LOG_WARNING, + "%s: zero client", __FUNCTION__); + return; + } + if (HA_OK != ha_msg_mod(msg,F_LRM_APP,client->app_name)) { + lrmd_log(LOG_ERR,"%s:%d: cannot add field to a message" + , __FUNCTION__, __LINE__); + return; + } + send_cbk_msg(msg, client); +} + +void +notify_client(lrmd_op_t* op) +{ + lrmd_client_t* client = lookup_client(op->client_id); + + if (client) { + /* send the result to client */ + send_cbk_msg(op->msg, client); + } else { + lrmd_log(LOG_WARNING + , "%s: client for the operation %s does not exist" + " and client requested notification." + , __FUNCTION__, op_info(op)); + } +} + +lrmd_client_t* +lookup_client (pid_t pid) +{ + return (lrmd_client_t*) g_hash_table_lookup(clients, &pid); +} + +static gboolean +client_cmp_name(gpointer key, gpointer val, gpointer app_name) +{ + return strcmp(((lrmd_client_t*)val)->app_name,(char *)app_name) ? + FALSE : TRUE; +} + +static lrmd_client_t* +lookup_client_by_name(char *app_name) +{ + return (lrmd_client_t*)g_hash_table_find(clients,client_cmp_name,app_name); +} + +lrmd_rsc_t* +lookup_rsc (const char* rid) +{ + return rid ? + (lrmd_rsc_t*)g_hash_table_lookup(resources, rid) : + NULL; +} + +lrmd_rsc_t* +lookup_rsc_by_msg (struct ha_msg* msg) +{ + const char* id = NULL; + lrmd_rsc_t* rsc = NULL; + + CHECK_ALLOCATED(msg, "msg", NULL); + id = ha_msg_value(msg, F_LRM_RID); + if (id == NULL) { + lrmd_log(LOG_ERR, "lookup_rsc_by_msg: got a NULL resource id."); + return NULL; + } + if (RID_LEN <= strnlen(id, RID_LEN+2)) { + lrmd_log(LOG_ERR, "lookup_rsc_by_msg: resource id is too long."); + return NULL; + } + rsc = lookup_rsc(id); + return rsc; +} + +static void +destroy_pipe_ra_stdout(gpointer user_data) +{ + ra_pipe_op_t * rapop = (ra_pipe_op_t *)user_data; + + CHECK_ALLOCATED(rapop, "ra_pipe_op",); + if (rapop->ra_stderr_fd < 0) { + ra_pipe_op_destroy(rapop); + } +} + +static void +destroy_pipe_ra_stderr(gpointer user_data) +{ + ra_pipe_op_t * rapop = (ra_pipe_op_t *)user_data; + + CHECK_ALLOCATED(rapop, "ra_pipe_op",); + if (rapop->ra_stdout_fd < 0) { + ra_pipe_op_destroy(rapop); + } +} + +static gboolean +handle_pipe_ra_stdout(int fd, gpointer user_data) +{ + gboolean rc = TRUE; + ra_pipe_op_t * rapop = (ra_pipe_op_t *)user_data; + char * data = NULL; + lrmd_op_t* lrmd_op = NULL; + + CHECK_ALLOCATED(rapop, "ra_pipe_op", FALSE); + + if (rapop->lrmd_op == NULL) { + lrmd_debug2(LOG_DEBUG, "%s:%d: Unallocated lrmd_op 0x%lx!!" + , __FUNCTION__, __LINE__ + , (unsigned long)rapop->lrmd_op); + } else { + lrmd_op = rapop->lrmd_op; + } + + if (fd <= STDERR_FILENO) { + lrmd_log(LOG_CRIT, "%s:%d: Attempt to read from " + "closed/invalid file descriptor %d." + , __FUNCTION__, __LINE__, fd); + return FALSE; + } + + if (0 != read_pipe(fd, &data, rapop)) { + /* error or reach the EOF */ + if (fd > STDERR_FILENO) { + close(fd); + if (fd == rapop->ra_stdout_fd) { + rapop->ra_stdout_fd = -1; + } + } + if ( NULL != rapop->ra_stdout_gsource) { + /* + * Returning FALSE will trigger ipc code to release + * the GFDSource, so donn't release it here. + */ + rapop->ra_stdout_gsource = NULL; + } + rc = FALSE; + } + + if ( data!=NULL ) { + if ( (0==STRNCMP_CONST(rapop->op_type, "meta-data")) + ||(0==STRNCMP_CONST(rapop->op_type, "monitor")) + ||(0==STRNCMP_CONST(rapop->op_type, "status")) ) { + lrmd_debug(LOG_DEBUG, "RA output: (%s:%s:stdout) %s" + , lrm_str(rapop->rsc_id), rapop->op_type, data); + } else { + lrmd_log(LOG_INFO, "RA output: (%s:%s:stdout) %s" + , lrm_str(rapop->rsc_id), rapop->op_type, data); + } + + /* + * This code isn't good enough, it produces erratic and hard-to + * read messages in the logs. But this does not affect the + * function correctness, since the first line output is ensured + * to be collected into the buffer completely. + * Anyway, the meta-data (which is _many_ lines long) can be + * handled by another function, see raexec.h + */ + if ( (rapop->first_line_read == FALSE) + && (0==STRNCMP_CONST(rapop->rsc_class, "heartbeat")) + && ( lrmd_op != NULL ) + && ( (0==STRNCMP_CONST(rapop->op_type, "monitor")) + ||(0==STRNCMP_CONST(rapop->op_type, "status")) )) { + if (lrmd_op != NULL) { + strncat(lrmd_op->first_line_ra_stdout, data + , sizeof(lrmd_op->first_line_ra_stdout) - + strlen(lrmd_op->first_line_ra_stdout)-1); + if (strchr(lrmd_op->first_line_ra_stdout, '\n') + != NULL) { + rapop->first_line_read = TRUE; + } + } else { + lrmd_log(LOG_CRIT + , "Before read the first line, the RA " + "execution child quitted and waited."); + } + } + + g_free(data); + } + + return rc; +} + +static gboolean +handle_pipe_ra_stderr(int fd, gpointer user_data) +{ + gboolean rc = TRUE; + char * data = NULL; + ra_pipe_op_t * rapop = (ra_pipe_op_t *)user_data; + + CHECK_ALLOCATED(rapop, "ra_pipe_op", FALSE); + + if (fd <= STDERR_FILENO) { + lrmd_log(LOG_CRIT, "%s:%d: Attempt to read from " + " closed/invalid file descriptor %d." + , __FUNCTION__, __LINE__, fd); + return FALSE; + } + + if (0 != read_pipe(fd, &data, rapop)) { + /* error or reach the EOF */ + if (fd > STDERR_FILENO) { + close(fd); + if (fd == rapop->ra_stderr_fd) { + rapop->ra_stderr_fd = -1; + } + } + if ( NULL != rapop->ra_stderr_gsource) { + /* + * G_main_del_fd will trigger + * destroy_pipe_ra_stderr + * ra_pipe_op_destroy + * + * Returning FALSE will trigger ipc code to release + * the GFDSource, so donn't release it here. + */ + rapop->ra_stderr_gsource = NULL; + } + rc = FALSE; + } + + if (data!=NULL) { + lrmd_log(LOG_INFO, "RA output: (%s:%s:stderr) %s" + , lrm_str(rapop->rsc_id), probe_str(rapop->lrmd_op,rapop->op_type), data); + g_free(data); + } + + return rc; +} + +int +read_pipe(int fd, char ** data, void * user_data) +{ + const int BUFFLEN = 81; + char buffer[BUFFLEN]; + int readlen; + GString * gstr_tmp; + int rc = 0; + lrmd_op_t * op = NULL; + ra_pipe_op_t * rapop = (ra_pipe_op_t *)user_data; + + lrmd_debug3(LOG_DEBUG, "%s begin.", __FUNCTION__); + + CHECK_ALLOCATED(rapop, "ra_pipe_op", FALSE); + + op = (lrmd_op_t *)rapop->lrmd_op; + if (NULL == op) { + lrmd_debug2(LOG_DEBUG, "%s:%d: Unallocated lrmd_op 0x%lx!!" + , __FUNCTION__, __LINE__ + , (unsigned long)op); + } + + *data = NULL; + gstr_tmp = g_string_new(""); + + do { + errno = 0; + readlen = read(fd, buffer, BUFFLEN - 1); + if (NULL == op) { + lrmd_debug2(LOG_NOTICE + , "read's ret: %d when lrmd_op finished" + , readlen); + } + if ( readlen > 0 ) { + buffer[readlen] = EOS; + g_string_append(gstr_tmp, buffer); + } + } while (readlen == BUFFLEN - 1 || errno == EINTR); + + if (errno == EINTR || errno == EAGAIN) { + errno = 0; + } + + /* Reach the EOF */ + if (readlen == 0) { + rc = -1; + } + + if ((readlen < 0) && (errno !=0)) { + rc = -1; + switch (errno) { + default: + cl_perror("%s:%d read error: fd %d errno=%d" + , __FUNCTION__, __LINE__ + , fd, errno); + if (NULL != op) { + lrmd_op_dump(op, "op w/bad errno"); + } else { + lrmd_log(LOG_NOTICE + , "%s::%d: lrmd_op has been freed" + , __FUNCTION__, __LINE__); + } + break; + + case EBADF: + lrmd_log(LOG_CRIT + , "%s:%d" + " Attempt to read from closed file descriptor %d." + , __FUNCTION__, __LINE__, fd); + if (NULL != op) { + lrmd_op_dump(op, "op w/bad errno"); + } else { + lrmd_log(LOG_NOTICE + , "%s::%d: lrmd_op has been freed" + , __FUNCTION__, __LINE__); + } + break; + } + } + + if ( gstr_tmp->len == 0 ) { + g_string_free(gstr_tmp, TRUE); + } else { + *data = gstr_tmp->str; + g_string_free(gstr_tmp, FALSE); + } + + lrmd_debug3(LOG_DEBUG, "%s end.", __FUNCTION__); + return rc; +} + + +static gboolean +debug_level_adjust(int nsig, gpointer user_data) +{ + char s[16]; + + switch (nsig) { + case SIGUSR1: + debug_level++; + dump_data_for_debug(); + break; + + case SIGUSR2: + dump_data_for_debug(); + debug_level--; + if (debug_level < 0) { + debug_level = 0; + } + break; + + default: + lrmd_log(LOG_WARNING, "debug_level_adjust: Received an " + "unexpected signal(%d). Something wrong?.",nsig); + } + + snprintf(s, sizeof(s), "%d", debug_level); + setenv(HADEBUGVAL, s, 1); + return TRUE; +} + +static void +dump_data_for_debug(void) +{ + lrmd_debug(LOG_DEBUG, "begin to dump internal data for debugging."); + lrmd_dump_all_clients(); + lrmd_dump_all_resources(); + lrmd_debug(LOG_DEBUG, "end to dump internal data for debugging."); +} + +const char* +gen_op_info(const lrmd_op_t* op, gboolean add_params) +{ + static char info[512]; + lrmd_rsc_t* rsc = NULL; + const char * op_type; + GString * param_gstr; + GHashTable* op_params = NULL; + + if (NULL == op) { + lrmd_log(LOG_ERR, "%s:%d: op==NULL" + , __FUNCTION__, __LINE__); + return NULL; + } + rsc = lookup_rsc(op->rsc_id); + op_type = ha_msg_value(op->msg, F_LRM_OP); + + if (rsc == NULL) { + snprintf(info,sizeof(info) + ,"operation %s[%d] on unknown rsc(maybe deleted) for client %d" + ,lrm_str(op_type) + ,op->call_id ,op->client_id); + + }else{ + if (op->exec_pid > 1) { + snprintf(info, sizeof(info) + ,"operation %s[%d] with pid %d on %s for client %d" + ,lrm_str(op_type), op->call_id, op->exec_pid, lrm_str(rsc->id) + ,op->client_id); + } else { + snprintf(info, sizeof(info) + ,"operation %s[%d] on %s for client %d" + ,lrm_str(op_type), op->call_id, lrm_str(rsc->id) + ,op->client_id); + } + + if( add_params ) { + param_gstr = g_string_new(""); + op_params = ha_msg_value_str_table(op->msg, F_LRM_PARAM); + hash_to_str(op_params, param_gstr); + if (op_params) { + free_str_table(op_params); + op_params = NULL; + } + + snprintf(info+strlen(info), sizeof(info)-strlen(info) + ,", its parameters: %s",param_gstr->str); + + g_string_free(param_gstr, TRUE); + } + } + return info; +} + +static void +hash_to_str(GHashTable * params , GString * str) +{ + if (params) { + g_hash_table_foreach(params, hash_to_str_foreach, str); + } +} + +static void +hash_to_str_foreach(gpointer key, gpointer value, gpointer user_data) +{ + char buffer_tmp[80]; + GString * str = (GString *)user_data; + + g_snprintf(buffer_tmp, sizeof(buffer_tmp), "%s=[%s] " + , (char *)key, (char *)value); + str = g_string_append(str, buffer_tmp); +} + +static void +check_queue_duration(lrmd_op_t* op) +{ + unsigned long t_stay_in_list = 0; + static struct msg_ctrl *ml; + + CHECK_ALLOCATED(op, "op", ); + t_stay_in_list = longclockto_ms(op->t_perform - op->t_addtolist); + if ( t_stay_in_list > WARNINGTIME_IN_LIST) + { + if (!ml) + ml = cl_limit_log_new(logmsg_ctrl_defs + OP_STAYED_TOO_LONG); + cl_limit_log(ml, LOG_WARNING + , "perform_ra_op: the %s stayed in operation " + "list for %lu ms (longer than %d ms)" + , small_op_info(op), t_stay_in_list + , WARNINGTIME_IN_LIST + ); + if (debug_level >= 2) { + dump_data_for_debug(); + } + } +} + diff --git a/lrm/lrmd/lrmd.h b/lrm/lrmd/lrmd.h new file mode 100644 index 0000000..eadea88 --- /dev/null +++ b/lrm/lrmd/lrmd.h @@ -0,0 +1,282 @@ +#define MAX_PID_LEN 256 +#define MAX_PROC_NAME 256 +#define MAX_MSGTYPELEN 32 +#define MAX_CLASSNAMELEN 32 +#define WARNINGTIME_IN_LIST 10000 +#define OPTARGS "skrhvmi:" +#define PID_FILE HA_VARRUNDIR"/lrmd.pid" +#define LRMD_COREDUMP_ROOT_DIR HA_COREDIR +#define APPHB_WARNTIME_FACTOR 3 +#define APPHB_INTVL_DETLA 30 /* Millisecond */ + +#define lrmd_log(priority, fmt...); \ + cl_log(priority, fmt); + +#define lrmd_debug(priority, fmt...); \ + if ( debug_level >= 1 ) { \ + cl_log(priority, fmt); \ + } + +#define lrmd_debug2(priority, fmt...); \ + if ( debug_level >= 2 ) { \ + cl_log(priority, fmt); \ + } + +#define lrmd_debug3(priority, fmt...); \ + if ( debug_level >= 3 ) { \ + cl_log(priority, fmt); \ + } + +#define lrmd_nullcheck(p) ((p) ? (p) : "<null>") +#define lrm_str(p) (lrmd_nullcheck(p)) + +#define CHECK_ALLOCATED(thing, name, result) \ + if (!thing) { \ + lrmd_log(LOG_ERR \ + , "%s: %s pointer 0x%lx is not allocated." \ + , __FUNCTION__, name, (unsigned long)thing); \ + if (!in_alloc_dump) { \ + in_alloc_dump = TRUE; \ + dump_data_for_debug(); \ + in_alloc_dump = FALSE; \ + return result; \ + } \ + } + +#define CHECK_RETURN_OF_CREATE_LRM_RET do { \ + if (NULL == msg) { \ + lrmd_log(LOG_ERR \ + , "%s: cannot create a ret message with create_lrm_ret." \ + , __FUNCTION__); \ + return HA_FAIL; \ + } \ +} while(0) + +#define LOG_FAILED_TO_GET_FIELD(field) \ + lrmd_log(LOG_ERR \ + , "%s:%d: cannot get field %s from message." \ + ,__FUNCTION__,__LINE__,field) + +#define LOG_FAILED_TO_ADD_FIELD(field) \ + lrmd_log(LOG_ERR \ + , "%s:%d: cannot add the field %s to a message." \ + , __FUNCTION__ \ + , __LINE__ \ + , field) + +/* NB: There's a return in these macros, hence the names */ +#define return_on_no_int_value(msg,fld,i) do { \ + if (HA_OK != ha_msg_value_int(msg,fld,i)) { \ + LOG_FAILED_TO_GET_FIELD(fld); \ + return HA_FAIL; \ + } \ +} while(0) +#define return_on_no_value(msg,fld,v) do { \ + v = ha_msg_value(msg,fld); \ + if (!v) { \ + LOG_FAILED_TO_GET_FIELD(fld); \ + return HA_FAIL; \ + } \ +} while(0) + +#define LRMD_APPHB_HB \ + if (reg_to_apphb == TRUE) { \ + if (apphb_hb() != 0) { \ + reg_to_apphb = FALSE; \ + } \ + } + +#define tm2age(tm) \ + (cmp_longclock(tm, zero_longclock) <= 0) ? \ + 0 : longclockto_ms(sub_longclock(now, tm)) +#define tm2unix(tm) \ + (time(NULL)-(tm2age(tm)+999)/1000) + +/* + * The basic objects in our world: + * + * lrmd_client_t: + * Client - a process which has connected to us for service. + * + * lrmd_rsc_t: + * Resource - an abstract HA cluster resource implemented by a + * resource agent through our RA plugins + * It has two list of operations (lrmd_op_t) associated with it + * op_list - operations to be run as soon as they're ready + * repeat_op_list - operations to be run later + * It maintains the following tracking structures: + * last_op_done Last operation performed on this resource + * last_op_table Last operations of each type done per client + * + * lrmd_op_t: + * Resource operation - an operation on a resource -- requested + * by a client. + * + * ProcTrack - tracks a currently running resource operation. + * It points back to the lrmd_op_t that started it. + * + * Global structures containing these things: + * + * clients - a hash table of all (currently connected) clients + * + * resources - a hash table of all (currently configured) resources + * + * Proctrack keeps its own private data structures to keep track of + * child processes that it created. They in turn point to the + * lrmd_op_t objects that caused us to fork the child process. + * + * + */ + +/* + * Recognized privilege levels + */ + +#define PRIV_ADMIN 8 /* ADMIN_UIDS are administrators */ +#define ADMIN_UIDS "0,"HA_CCMUSER +#define ADMIN_GIDS "0,"HA_APIGROUP /* unused */ + +typedef struct +{ + char* app_name; + pid_t pid; + gid_t gid; + uid_t uid; + + IPC_Channel* ch_cmd; + IPC_Channel* ch_cbk; + + GCHSource* g_src; + GCHSource* g_src_cbk; + char lastrequest[MAX_MSGTYPELEN]; + time_t lastreqstart; + time_t lastreqend; + time_t lastrcsent; + int priv_lvl; /* client privilege level (depends on uid/gid) */ +}lrmd_client_t; + +typedef struct lrmd_rsc lrmd_rsc_t; +typedef struct lrmd_op lrmd_op_t; +typedef struct ra_pipe_op ra_pipe_op_t; + +#define RSC_REMOVAL_PENDING 1 +#define RSC_FLUSHING_OPS 2 +#define rsc_frozen(r) \ + ((r)->state==RSC_REMOVAL_PENDING || (r)->state==RSC_FLUSHING_OPS) +#define rsc_removal_pending(r) \ + ((r)->state==RSC_REMOVAL_PENDING) +#define set_rsc_removal_pending(r) \ + (r)->state = RSC_REMOVAL_PENDING +#define set_rsc_flushing_ops(r) \ + (r)->state = RSC_FLUSHING_OPS +#define rsc_reset_state(r) (r)->state = 0 +/* log messages for repeating ops (monitor) once an hour */ +#define LOGMSG_INTERVAL (60*60) +#define is_logmsg_due(op) \ + (longclockto_ms(sub_longclock(time_longclock(), op->t_lastlogmsg))/1000 >= \ + (unsigned long)LOGMSG_INTERVAL) +#define probe_str(op,op_type) \ + ((op && !op->interval && !strcmp(op_type,"monitor")) ? "probe" : op_type) +/* exclude stonith class from child count */ +#define no_child_count(rsc) \ + (strcmp((rsc)->class,"stonith") == 0) + +struct lrmd_rsc +{ + char* id; /* Unique resource identifier */ + char* type; /* */ + char* class; /* */ + char* provider; /* Resource provider (optional) */ + GHashTable* params; /* Parameters to this resource */ + /* as name/value pairs */ + GList* op_list; /* Queue of operations to run */ + GList* repeat_op_list; /* Unordered list of repeating */ + /* ops They will run later */ + GHashTable* last_op_table; /* Last operation of each type */ + lrmd_op_t* last_op_done; /* The last finished op of the resource */ + guint delay_timeout; /* The delay value of op_list execution */ + int state; /* status of the resource */ +}; + +struct lrmd_op +{ + char* rsc_id; + gboolean is_copy; + pid_t client_id; + int call_id; + int exec_pid; + guint repeat_timeout_tag; + int interval; + int delay; + gboolean is_cancelled; + int weight; + int copyparams; + struct ha_msg* msg; + ra_pipe_op_t * rapop; + char first_line_ra_stdout[80]; /* only for heartbeat RAs*/ + /*time stamps*/ + longclock_t t_recv; /* set in lrmd_op_new(), i.e. on op create */ + longclock_t t_addtolist; /* set in add_op_to_runlist() */ + longclock_t t_perform; /* set in perform_ra_op() */ + longclock_t t_done; /* set in on_op_done() */ + longclock_t t_rcchange; /* set in on_op_done(), could equal t_perform */ + longclock_t t_lastlogmsg; /* the last time the monitor op was logged */ + ProcTrackKillInfo killseq[3]; +}; + + +/* For reading the output from executing the RA */ +struct ra_pipe_op +{ + /* The same value of the one in corresponding lrmd_op */ + lrmd_op_t * lrmd_op; + int ra_stdout_fd; + int ra_stderr_fd; + GFDSource * ra_stdout_gsource; + GFDSource * ra_stderr_gsource; + gboolean first_line_read; + + /* For providing more detailed information in log */ + char * rsc_id; + char * op_type; + char * rsc_class; +}; + + +const char *gen_op_info(const lrmd_op_t* op, gboolean add_params); +#define op_info(op) gen_op_info(op,TRUE) +#define small_op_info(op) gen_op_info(op,FALSE) + +#define DOLRMAUDITS +#undef DOLRMAUDITS + +#define DOMEGALRMAUDITS +#define LRMAUDIT_CLIENTS +#define LRMAUDIT_RESOURCES + +#ifdef DOLRMAUDITS + + void lrmd_audit(const char *function, int line); + void audit_clients(void); + void audit_resources(void); + void audit_ops(GList* rsc_ops, lrmd_rsc_t *rsc, const char *desc); + void on_client(gpointer key, gpointer value, gpointer user_data); + void on_resource(gpointer key, gpointer value, gpointer user_data); + void on_op(lrmd_op_t *op, lrmd_rsc_t *rsc, const char *desc); + void on_ra_pipe_op(ra_pipe_op_t *rapop, lrmd_op_t *op, const char *desc); + +# define LRMAUDIT() lrmd_audit(__FUNCTION__,__LINE__) +# ifdef DOMEGALRMAUDITS +# define MEGALRMAUDIT lrmd_audit(__FUNCTION__,__LINE__) +# else +# define MEGALRMAUDIT /*nothing*/ +# endif +#else +# define LRMAUDIT() /*nothing*/ +# define MEGALRMAUDIT() /*nothing*/ +#endif + +/* + * load parameters from an ini file (cib_secrets.c) + */ +int replace_secret_params(char* rsc_id, GHashTable* params); diff --git a/lrm/lrmd/lrmd_fdecl.h b/lrm/lrmd/lrmd_fdecl.h new file mode 100644 index 0000000..9c97385 --- /dev/null +++ b/lrm/lrmd/lrmd_fdecl.h @@ -0,0 +1,111 @@ +/* TODO: This ought to be broken up into several source files for easier + * reading and debugging. */ + +/* Debug oriented funtions */ +static gboolean debug_level_adjust(int nsig, gpointer user_data); +static void dump_data_for_debug(void); + +/* glib loop call back functions */ +static gboolean on_connect_cmd(IPC_Channel* ch_cmd, gpointer user_data); +static gboolean on_connect_cbk(IPC_Channel* ch_cbk, gpointer user_data); +static int msg_type_cmp(const void *p1, const void *p2); +static gboolean on_receive_cmd(IPC_Channel* ch_cmd, gpointer user_data); +static gboolean on_repeat_op_readytorun(gpointer data); +static void on_remove_client(gpointer user_data); +static void destroy_pipe_ra_stderr(gpointer user_data); +static void destroy_pipe_ra_stdout(gpointer user_data); + +/* message handlers */ +static int on_msg_register(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_rsc_classes(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_rsc_types(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_rsc_providers(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_metadata(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_add_rsc(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_rsc(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_last_op(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_all(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_del_rsc(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_fail_rsc(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_cancel_op(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_flush_all(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_perform_op(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_state(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_set_lrmd_param(lrmd_client_t* client, struct ha_msg* msg); +static int on_msg_get_lrmd_param(lrmd_client_t* client, struct ha_msg* msg); +static int set_lrmd_param(const char *name, const char *value); +static int get_lrmd_param(const char *name, char *value, int maxstring); +static gboolean sigterm_action(int nsig, gpointer unused); + +/* functions wrap the call to ra plugins */ +static int perform_ra_op(lrmd_op_t* op); + +/* Apphb related functions */ +static int init_using_apphb(void); +static gboolean emit_apphb(gpointer data); + +/* Utility functions */ +static int flush_op(lrmd_op_t* op); +static gboolean rsc_execution_freeze_timeout(gpointer data); +static void add_op_to_runlist(lrmd_rsc_t* rsc, lrmd_op_t* op); +static int perform_op(lrmd_rsc_t* rsc); +static int unregister_client(lrmd_client_t* client); +static int on_op_done(lrmd_rsc_t* rsc, lrmd_op_t* op); +static int send_ret_msg ( IPC_Channel* ch, int rc); +static void send_cbk_msg(struct ha_msg* msg, lrmd_client_t* client); +static void send_msg(struct ha_msg* msg, lrmd_client_t* client); +static void notify_client(lrmd_op_t* op); +static lrmd_client_t* lookup_client (pid_t pid); +static lrmd_rsc_t* lookup_rsc (const char* rid); +static lrmd_rsc_t* lookup_rsc_by_msg (struct ha_msg* msg); +static int read_pipe(int fd, char ** data, gpointer user_data); +static gboolean handle_pipe_ra_stdout(int fd, gpointer user_data); +static gboolean handle_pipe_ra_stderr(int fd, gpointer user_data); +static struct ha_msg* op_to_msg(lrmd_op_t* op); +static int store_timestamps(lrmd_op_t* op); +static void reset_timestamps(lrmd_op_t* op); +static gboolean lrm_shutdown(void); +static gboolean can_shutdown(void); +static gboolean free_str_hash_pair(gpointer key +, gpointer value, gpointer user_data); +static gboolean free_str_op_pair(gpointer key +, gpointer value, gpointer user_data); +static lrmd_op_t* lrmd_op_copy(const lrmd_op_t* op); +static void send_last_op(gpointer key, gpointer value, gpointer user_data); +static void replace_last_op(lrmd_client_t* client, lrmd_rsc_t* rsc, lrmd_op_t* op); +static int record_op_completion(lrmd_rsc_t* rsc, lrmd_op_t* op); +static void to_repeatlist(lrmd_rsc_t* rsc, lrmd_op_t* op); +static void remove_op_history(lrmd_op_t* op); +static void hash_to_str(GHashTable * , GString *); +static void hash_to_str_foreach(gpointer key, gpointer value, gpointer userdata); +static void warning_on_active_rsc(gpointer key, gpointer value, gpointer user_data); +static void check_queue_duration(lrmd_op_t* op); +static gboolean flush_all(GList** listp, int client_pid); +static gboolean cancel_op(GList** listp,int cancel_op_id); +static int prepare_failmsg(struct ha_msg* msg, + int fail_rc, const char *fail_reason); +static void async_notify(gpointer key, gpointer val, gpointer data); +static gboolean client_cmp_name(gpointer key, gpointer val, gpointer app_name); +static lrmd_client_t* lookup_client_by_name(char *app_name); +static void calc_max_children(void); + +/* + * following functions are used to monitor the exit of ra proc + */ +static void on_ra_proc_registered(ProcTrack* p); +static void on_ra_proc_finished(ProcTrack* p, int status +, int signo, int exitcode, int waslogged); +static const char* on_ra_proc_query_name(ProcTrack* p); + + + +/* + * Daemon functions + * + * copy from the code of Andrew Beekhof <andrew@beekhof.net> + */ +static void usage(const char* cmd, int exit_status); +static int init_start(void); +static int init_stop(const char *pid_file); +static int init_status(const char *pid_file, const char *client_name); +static void lrmd_rsc_dump(char* rsc_id, const char * text); diff --git a/lrm/test/LRMBasicSanityCheck.in b/lrm/test/LRMBasicSanityCheck.in new file mode 100755 index 0000000..dbe8548 --- /dev/null +++ b/lrm/test/LRMBasicSanityCheck.in @@ -0,0 +1,55 @@ +#!/bin/sh + + # Copyright (c) 2004 International Business Machines + # Author: Huang Zhen <zhenhltc@cn.ibm.com> + # + # This program is free software; you can redistribute it and/or + # modify it under the terms of the GNU General Public + # License as published by the Free Software Foundation; either + # version 2.1 of the License, or (at your option) any later version. + # + # This software is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + # General Public License for more details. + # + # You should have received a copy of the GNU General Public + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + # +HBLIB=@libdir@/heartbeat +LRMD=$HBLIB/lrmd +LRMADMIN=@sbindir@/lrmadmin +export LRMD LRMADMIN + +if [ $# -gt 0 ]; then + LRMD=$1/lrmd +fi + +if [ ! -f $LRMD ]; then + echo $LRMD does not exist + exit 1 +fi + +if [ ! -f $LRMADMIN ]; then + echo $LRMADMIN does not exist + exit 1 +fi + +OUTDIR=/tmp/LRM_BSC_$$ +export OUTDIR +[ -d $OUTDIR ] && { + echo $OUTDIR exists, please cleanup + exit 1 +} + +`dirname $0`/regression.sh -q set:BSC +rc=$? +if [ $rc -eq 0 ]; then + echo "LRM tests PASSED" + rm -rf $OUTDIR +else + echo "LRM tests FAILED" + echo "Please check $OUTDIR for results" +fi +exit $rc diff --git a/lrm/test/Makefile.am b/lrm/test/Makefile.am new file mode 100644 index 0000000..84f6657 --- /dev/null +++ b/lrm/test/Makefile.am @@ -0,0 +1,48 @@ +# +# Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright (c) 2004 International Business Machines +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +SUBDIRS = testcases + +INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl + +COMMONLIBS = $(top_builddir)/lib/clplumbing/libplumb.la $(GLIBLIB) + +noinst_PROGRAMS = apitest plugintest callbacktest + +apitest_SOURCES = apitest.c +apitest_LDFLAGS = $(COMMONLIBS) +apitest_LDADD = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la +apitest_DEPENDENCIES = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la + +plugintest_SOURCES = plugintest.c +plugintest_LDADD = $(COMMONLIBS) +plugintest_LDFLAGS = -L$(top_builddir)/lib/pils -lpils @LIBLTDL@ + +callbacktest_SOURCES = callbacktest.c +callbacktest_LDFLAGS = $(COMMONLIBS) +callbacktest_LDADD = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la +callbacktest_DEPENDENCIES = $(top_builddir)/lib/$(LRM_DIR)/liblrm.la + +testdir = $(datadir)/$(PACKAGE_NAME)/lrmtest +test_SCRIPTS = LRMBasicSanityCheck regression.sh evaltest.sh lrmregtest lrmregtest-lsb +test_DATA = README.regression defaults descriptions lrmadmin-interface language +# shouldn't need this, but we do for some versions of autofoo tools +EXTRA_DIST = $(test_SCRIPTS) $(test_DATA) diff --git a/lrm/test/README.regression b/lrm/test/README.regression new file mode 100644 index 0000000..3588172 --- /dev/null +++ b/lrm/test/README.regression @@ -0,0 +1,164 @@ +LRM regression tests + +* WARNING * WARNING * WARNING * WARNING * WARNING * WARNING * +* +* evaltest.sh uses eval to an extent you don't really want to +* know about. Beware. Beware twice. Any input from the testcases +* directory is considered to be trusted. So, think twice before +* devising your tests lest you kill your precious data. Got it? +* Good. +* +* Furthermore, we are deliberately small on testing the user +* input and no one should try to predict what is to happen on +* random input from the testcases. +* +* WARNING * WARNING * WARNING * WARNING * WARNING * WARNING * + +Manifest + + regression.sh: the top level program + evaltest.sh: the engine test engine + + lrmadmin-interface: interface to lrmd (lrmadmin) + descriptions: describe what we are about to do + defaults: the default settings for test commands + + testcases/: here are the testcases and filters + output/: here goes the output + +All volatile data lives in the testcases/ directory. + +NB: You should never ever need to edit regression.sh and +evaltest.sh. If you really have to, please talk to me and I will +try to fix it so that you do not have to. + +Please write new test cases. The more the merrier :) + +Usage + +The usage is: + + ./regression.sh ["prepare"] ["set:"<setname>|<testcase>] + +Test cases are collected in test sets. The default test set is +basicset and running regression.sh without arguments will do all +tests from that set. + +To show progress, for each test a '.' is printed. For sleeps, +a '+' for each second. Once all tests have been evaluated, the +output is checked against the expect file. If successful, "PASS" +is printed, otherwise "FAIL". + +Specifying "prepare" will make regression.sh create expect +output files for the given set of tests or testcase. + +The script will start and stop lrmd itself. stonithd is also +started to test the XML descriptions printed by stonith agents. +No other parts of stonithd functionality is tested. + +The following files may be generated: + + output/<testcase>.out: the output of the testcase + output/regression.out: the output of regression.sh + output/lrmd.out: the output of lrmd + +On success output from testcases is removed and regression.out is +empty. + +Driving the test cases yourself + +evaltest.sh accepts input from stdin, evaluates it immediately, +and prints results to stdout/stderr. One can perhaps get a better +feeling of what's actually going on by running it interactively. +Please note that you have to start the lrmd yourself beforehand. + +Test cases + +Tests are written in a simple metalanguage. The list of commands +with rough translation to lrmadmin's options is in the language +file. The best description of the language is in the +lrmadmin-interface and descriptions scripts: + +$ egrep '^lrm|echo' lrmadmin-interface descriptions + +A test case is a list of tests, one per line. A few examples: + + add # add a resource with default name + list # list all resources + del rsc=wiwi # remove a wiwi resource + +A set of defaults for LRM options is in the defaults file. That's +why we can write short forms instead of + + add rsc=r1 class=ocf type=lrmregtest provider=heartbeat ... + +Special operations + +There are special operations with which it is possible to change +environment and do other useful things. All special ops start +with the '%' sign and may be followed by additional parameters. + +%setenv + change the environment variable; see defaults for the + set of global variables and resetvars() in evaltest.sh + +%sleep + sleep + +%stop + skip the rest of the tests + +%extcheck + feed the output of the next test case to the specified + external program/filter; the program should either reside in + testcases/ or be in the PATH, i.e. + + %extcheck cat + + simulates a null op :) + + see testcases/metadata for some examples + +%repeat num + repeat the next test num times + there are several variables which are substituted in the test + lines, so that we can simulate a for loop: + + s/%t/$test_cnt/g + s/%l/$line/g + s/%j/$job_cnt/g + s/%i/$repeat_cnt/g + + for example, to add 10 resources: + + %repeat 10 + add rsc=r-%i + +%bg [num] + run next num (or just the next one) tests in background + +%bgrepeat [num] + a combination of the previous two (used often) + +%wait + wait for the last background test to finish + +%shell + feed whatever is in the rest of the line to 'sh -s' + +Filters and except files + +Some output is necessarily very volatile, such as time stamps. +It is possible to specify a filter for each testcase to get rid +of superfluous information. A filter is a filter in UNIX +sense, it takes input from stdin and prints results to stdout. + +There is a common filter called very inventively +testcases/common.filter which is applied to all test cases. + +Except files are a list of extended regular expressions fed to +egrep(1). That way one can filter out lines which are not +interesting. Again, the one applied to all is +testcases/common.excl. + + diff --git a/lrm/test/apitest.c b/lrm/test/apitest.c new file mode 100644 index 0000000..0d4c572 --- /dev/null +++ b/lrm/test/apitest.c @@ -0,0 +1,317 @@ + +/* + * Test program for Local Resource Manager API. + * + * Copyright (C) 2004 Huang Zhen <zhenh@cn.ibm.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include <lha_internal.h> +#include <unistd.h> +#include <stdio.h> +#include <sys/poll.h> +#include <string.h> +#include <glib.h> +#include <lrm/lrm_api.h> +#include <clplumbing/cl_log.h> +#include <syslog.h> + +void lrm_op_done_callback (lrm_op_t* op); +void printf_rsc(lrm_rsc_t* rsc); +void printf_op(lrm_op_t* op); +void printf_hash_table(GHashTable* hash_table); +void get_all_rsc(ll_lrm_t* lrm); +void get_cur_state(lrm_rsc_t* rsc); + +int main (int argc, char* argv[]) +{ + ll_lrm_t* lrm; + lrm_rsc_t* rsc = NULL; + lrm_op_t* op = NULL; + const char* rid = "ip248"; + GHashTable* param = NULL; + GList* classes; + int i; + + cl_log_set_entity("apitest"); + cl_log_set_facility(LOG_USER); + + lrm = ll_lrm_new("lrm"); + + if(NULL == lrm) + { + printf("lrm==NULL\n"); + return 1; + } + puts("sigon..."); + lrm->lrm_ops->signon(lrm,"apitest"); + + classes = lrm->lrm_ops->get_rsc_class_supported(lrm); + lrm_free_str_list(classes); + + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + puts("add_rsc..."); + lrm->lrm_ops->add_rsc(lrm, rid, "heartbeat", "IPaddr", "heartbeat", param); + puts("get_rsc..."); + rsc = lrm->lrm_ops->get_rsc(lrm, rid); + printf_rsc(rsc); + + puts("perform_op(start)..."); + op = lrm_op_new(); + op->op_type = g_strdup("start"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a start op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc = EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(status)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("status"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a status op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 1000; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(stop)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("stop"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a stop op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(status)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("status"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a status op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 2000; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(start)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("start"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a start op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc = EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(status)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("status"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a status op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 3000; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + puts("perform_op(stop)..."); + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("192.168.192.100")); + op = lrm_op_new(); + op->op_type = g_strdup("stop"); + op->params = param; + op->timeout = 0; + op->user_data = strdup("It is a stop op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + lrm_free_op(op); + + for(i = 0; i < 5; i++) { + puts("get_cur_state..."); + get_cur_state(rsc); + puts("sleep a while..."); + sleep(1); + } + + puts("delete_rsc..."); + lrm->lrm_ops->delete_rsc(lrm, rid); + lrm_free_rsc(rsc); + + puts("signoff..."); + lrm->lrm_ops->signoff(lrm); + + return 0; +} +void lrm_op_done_callback(lrm_op_t* op) +{ + puts("lrm_op_done_callback..."); + printf_op(op); +} +void printf_rsc(lrm_rsc_t* rsc) +{ + printf("print resource>>>>>>>>>\n"); + if (NULL == rsc) { + printf("resource is null\n"); + printf("print end\n"); + return; + } + printf("\tresource of id:%s\n", rsc->id); + printf("\ttype:%s\n", rsc->type); + printf("\tclass:%s\n", rsc->class); + printf("\tparams:\n"); + printf_hash_table(rsc->params); + printf("print end<<<<<<<<<<<<<<<\n"); +} + +void printf_op(lrm_op_t* op) +{ + printf("print op>>>>>>>>>>>>>>>>\n"); + + if (NULL == op) { + printf("op is null\n"); + printf("print end\n"); + return; + } + + printf("\top_type:%s\n",op->op_type?op->op_type:"null"); + printf("\tparams:\n"); + printf_hash_table(op->params); + printf("\ttimeout:%d\n",op->timeout); + printf("\tuser_data:%s\n",op->user_data?(char*)op->user_data:"null"); + printf("\top_status:%d\n",op->op_status); + printf("\tapp_name:%s\n",op->app_name?op->app_name:"null"); + printf("\toutput:%s\n",op->output?op->output:"null"); + printf("\trc:%d\n",op->rc); + printf("\tcall_id:%d\n",op->call_id); + printf("print end<<<<<<<<<<<<<<<<<<\n"); +} + +static void +printf_pair(gpointer key, gpointer value, gpointer user_data) +{ + printf("\t\t%s=%s\n",(char*)key,(char*)value); +} +void +printf_hash_table(GHashTable* hash_table) +{ + if (NULL == hash_table) { + printf("\t\tnull\n"); + return; + } + g_hash_table_foreach(hash_table, printf_pair, NULL); +} +void +get_all_rsc(ll_lrm_t* lrm) +{ + GList* element = NULL, * rid_list = NULL; + + puts("get_all_rscs..."); + rid_list = lrm->lrm_ops->get_all_rscs(lrm); + if (NULL != rid_list) { + element = g_list_first(rid_list); + while (NULL != element) { + printf("\tid:%s\n",(char*)element->data); + element = g_list_next(element); + } + } else { + puts("\tnone."); + } + lrm_free_str_list(rid_list); +} +void +get_cur_state(lrm_rsc_t* rsc) +{ + state_flag_t state; + GList* node = NULL, * op_list = NULL; + lrm_op_t* op = NULL; + printf("current state>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); + + op_list = rsc->ops->get_cur_state(rsc, &state); + + printf("\tcurrent state:%s\n",state==LRM_RSC_IDLE?"Idle":"Busy"); + + + for(node = g_list_first(op_list); NULL != node; + node = g_list_next(node)) { + op = (lrm_op_t*)node->data; + printf_op(op); + } + lrm_free_op_list(op_list); + printf("current end<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"); +} diff --git a/lrm/test/apitest.exp b/lrm/test/apitest.exp new file mode 100644 index 0000000..b153ee3 --- /dev/null +++ b/lrm/test/apitest.exp @@ -0,0 +1,122 @@ +sigon... +add_rsc... +get_rsc... +print resource + resource of id:ip248 + type:IPv6addr + class:heartbeat + params: + 1=3ffe:ffff:0:f101::3 +print end +perform_op(start)... +print op + op_type:start + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a start op! + op_status:0 + app_name:null + output:null + rc:0 +print end +perform_op(status)... +print op + op_type:status + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a status op! + op_status:0 + app_name:null + output:null + rc:0 +print end +perform_op(stop)... +print op + op_type:stop + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a stop op! + op_status:0 + app_name:null + output:null + rc:0 +print end +get_cur_state... + current state:Busy +print op + op_type:start + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a start op! + op_status:-1 + app_name:apitest + output:null + rc:0 +print end +print op + op_type:status + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a status op! + op_status:-1 + app_name:apitest + output:null + rc:0 +print end +print op + op_type:stop + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:It is a stop op! + op_status:-1 + app_name:apitest + output:null + rc:0 +print end +stop_op... +get_cur_state... + current state:Busy +print op + op_type:start + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:null + op_status:-1 + app_name:apitest + output:null + rc:0 +print end +print op + op_type:stop + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:null + op_status:-1 + app_name:apitest + output:null + rc:0 +print end +sleep a while... +get_cur_state... + current state:Idel +print op + op_type:stop + params: + 1=3ffe:ffff:0:f101::3 + timeout:0 + user_data:null + op_status:0 + app_name:apitest + output:null + rc:0 +print end +delete_rsc... +signoff... diff --git a/lrm/test/callbacktest.c b/lrm/test/callbacktest.c new file mode 100644 index 0000000..48f4d49 --- /dev/null +++ b/lrm/test/callbacktest.c @@ -0,0 +1,204 @@ + +/* + * Test program for the callback function of Local Resource Manager API. + * + * Copyright (C) 2004 Huang Zhen <zhenh@cn.ibm.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +#include <lha_internal.h> +#include <unistd.h> +#include <stdio.h> +#include <sys/poll.h> +#include <string.h> +#include <glib.h> +#include <lrm/lrm_api.h> +#include <syslog.h> +#include <clplumbing/GSource.h> + +static void lrm_op_done_callback(lrm_op_t *op); +static void printf_rsc(lrm_rsc_t *rsc); +static void printf_op(lrm_op_t *op); +static void printf_hash_table(GHashTable *hash_table); +static gboolean lrm_dispatch(IPC_Channel *notused, gpointer user_data); +static GMainLoop *mainloop; + +int +main(int argc, char *argv[]) +{ + ll_lrm_t* lrm; + lrm_rsc_t* rsc = NULL; + lrm_op_t* op = NULL; + const char* rid = "ip248"; + GHashTable* param = NULL; + + lrm = ll_lrm_new("lrm"); + + if(NULL == lrm) + { + printf("lrm==NULL\n"); + return 1; + } + puts("sigon..."); + lrm->lrm_ops->signon(lrm,"apitest"); + lrm->lrm_ops->set_lrm_callback(lrm, lrm_op_done_callback); + + param = g_hash_table_new(g_str_hash,g_str_equal); + g_hash_table_insert(param, strdup("1"), strdup("3ffe:ffff:0:f101::3")); + puts("add_rsc..."); + lrm->lrm_ops->add_rsc(lrm, rid, "heartbeat", "IPv6addr", NULL, param); + puts("get_rsc..."); + rsc = lrm->lrm_ops->get_rsc(lrm, rid); + printf_rsc(rsc); + + puts("perform_op(start)..."); + op = lrm_op_new(); + op->op_type = g_strdup("start"); + op->params = NULL; + op->timeout = 0; + op->user_data = strdup("It is a start op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc = EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + + puts("perform_op(status)..."); + op = lrm_op_new(); + op->op_type = g_strdup("status"); + op->params = NULL; + op->timeout = 0; + op->user_data = strdup("It is a status op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 1000; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + + puts("perform_op(stop)..."); + op = lrm_op_new(); + op->op_type = g_strdup("stop"); + op->params = NULL; + op->timeout = 0; + op->user_data = strdup("It is a stop op!"); + if ( op->user_data == NULL ) { + fprintf(stderr, "No enough memory.\n"); + return -1; + } + op->user_data_len = strlen(op->user_data)+1; + op->interval = 0; + op->target_rc=EVERYTIME; + rsc->ops->perform_op(rsc,op); + printf_op(op); + + G_main_add_IPC_Channel(G_PRIORITY_LOW, + lrm->lrm_ops->ipcchan(lrm), + FALSE, + lrm_dispatch, lrm, + NULL); + + mainloop = g_main_new(FALSE); + g_main_run(mainloop); + + puts("delete_rsc..."); + lrm->lrm_ops->delete_rsc(lrm, rid); + + puts("signoff..."); + lrm->lrm_ops->signoff(lrm); + + return 0; +} + +static void +lrm_op_done_callback(lrm_op_t *op) +{ + puts("lrm_op_done_callback..."); + printf_op(op); +} + +static gboolean +lrm_dispatch(IPC_Channel *notused, gpointer user_data) +{ + ll_lrm_t *lrm = (ll_lrm_t*)user_data; + lrm->lrm_ops->rcvmsg(lrm, FALSE); + return TRUE; +} + +static void +printf_rsc(lrm_rsc_t *rsc) +{ + printf("print resource\n"); + if (NULL == rsc) { + printf("resource is null\n"); + printf("print end\n"); + return; + } + printf("\tresource of id:%s\n", rsc->id); + printf("\ttype:%s\n", rsc->type); + printf("\tclass:%s\n", rsc->class); + printf("\tparams:\n"); + printf_hash_table(rsc->params); + printf("print end\n"); +} + +static void +printf_op(lrm_op_t *op) +{ + printf("print op\n"); + + if (NULL == op) { + printf("op is null\n"); + printf("print end\n"); + return; + } + + printf("\top_type:%s\n",op->op_type?op->op_type:"null"); + printf("\tparams:\n"); + printf_hash_table(op->params); + printf("\ttimeout:%d\n",op->timeout); + printf("\tuser_data:%s\n",op->user_data?(char*)op->user_data:"null"); + printf("\tuser_data pointer:%p\n",op->user_data); + printf("\top_status:%d\n",op->op_status); + printf("\tapp_name:%s\n",op->app_name?op->app_name:"null"); + printf("\toutput:%s\n",op->output?op->output:"null"); + printf("\trc:%d\n",op->rc); +/* printf("\tcall_id:%d\n",op->call_id); */ + printf("print end\n"); +} + +static void +printf_pair(gpointer key, gpointer value, gpointer user_data) +{ + printf("\t\t%s=%s\n",(char*)key,(char*)value); +} + +static void +printf_hash_table(GHashTable *hash_table) +{ + if (NULL == hash_table) { + printf("\t\tnull\n"); + return; + } + g_hash_table_foreach(hash_table, printf_pair, NULL); +} diff --git a/lrm/test/defaults b/lrm/test/defaults new file mode 100644 index 0000000..039915b --- /dev/null +++ b/lrm/test/defaults @@ -0,0 +1,9 @@ +# defaults +: ${dflt_rsc:=r1} +: ${dflt_type:=lrmregtest} +: ${dflt_class:=ocf} +: ${dflt_provider:=heartbeat} +: ${dflt_timeout:=1000} +: ${dflt_interval:=0} +: ${dflt_targetrc:=EVERYTIME} +dflt_args="" diff --git a/lrm/test/descriptions b/lrm/test/descriptions new file mode 100644 index 0000000..f2aab6b --- /dev/null +++ b/lrm/test/descriptions @@ -0,0 +1,55 @@ +lead=".TRY" +describe_list() { + echo $lead List resources +} +describe_add() { + echo $lead Add resource \ + ${rsc:-$dflt_rsc} \ + class=${class:-$dflt_class} type=${type:-$dflt_type} \ + provider=${provider:-$dflt_provider} \ + args=$args +} +describe_del() { + echo $lead Delete resource \ + ${rsc:-$dflt_rsc} +} +describe_flush() { + echo $lead Flush resource \ + ${rsc:-$dflt_rsc} +} +describe_state() { + echo $lead Show state \ + ${rsc:-$dflt_rsc} +} +describe_info() { + echo $lead Show info \ + ${rsc:-$dflt_rsc} +} +describe_exec() { + echo $lead Exec \ + ${rsc:-$dflt_rsc} \ + op=${operation:-$dflt_operation} \ + timeout=${timeout:-$dflt_timeout} interval=${interval:-$dflt_interval} \ + target=${targetrc:-$dflt_targetrc} args=$args +} + +describe_classes() { + echo $lead List classes +} +describe_types() { + echo $lead List types \ + class=${class:-$dflt_class} +} +describe_classmeta() { + echo $lead Meta-data \ + class=${class:-$dflt_class} +} +describe_meta() { + echo $lead Show meta-data \ + class=${class:-$dflt_class} \ + type=${type:-$dflt_type} provider=${provider:-$dflt_provider} +} +describe_provider() { + echo $lead Show provider \ + class=${class:-$dflt_class} type=${type:-$dflt_type} +} diff --git a/lrm/test/evaltest.sh b/lrm/test/evaltest.sh new file mode 100755 index 0000000..f369102 --- /dev/null +++ b/lrm/test/evaltest.sh @@ -0,0 +1,171 @@ +#!/bin/sh + + # Copyright (C) 2007 Dejan Muhamedagic <dejan@suse.de> + # + # This program is free software; you can redistribute it and/or + # modify it under the terms of the GNU General Public + # License as published by the Free Software Foundation; either + # version 2.1 of the License, or (at your option) any later version. + # + # This software is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + # General Public License for more details. + # + # You should have received a copy of the GNU General Public + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + # + +: ${TESTDIR:=testcases} +: ${LRMADMIN:=../admin/lrmadmin} +test -x $LRMADMIN || LRMADMIN=lrmadmin +: ${OCF_ROOT:=/usr/lib/ocf} + +. ./defaults +. ./lrmadmin-interface +. ./descriptions + +resetvars() { + unset rsc type class provider timeout interval targetrc args + unset extcheck +} + +# +# special operations squad +# +specopt_setenv() { + eval $rest +} +specopt_sleep() { + #sleep $rest + # the while loop below is the same + # but we give user some feedback on what's happening + while [ "$rest" -gt 0 ]; do + sleep 1 + echo -n "+" >&3 + rest=$(($rest-1)) + done +} +specopt_extcheck() { + extcheck="$rest" + set $extcheck + which "$1" >/dev/null 2>&1 || # a program in the PATH + extcheck="$TESTDIR/$extcheck" # or our script +} +specopt_repeat() { + repeat_limit=$rest +} +specopt_bg() { + if [ "$job_cnt" -gt "$bgprocs_num" ]; then + bgprocs_num=${rest:-1} + job_cnt=1 + else + echo ".BG bad usage: more tests yet to be backgrounded" + fi +} +specopt_bgrepeat() { # common + specopt_bg + specopt_repeat +} +specopt_wait() { # common + waitforbgprocs +} +specopt_shell() { # run command with shell + echo "$rest" | sh -s | # and execute the command + { [ "$extcheck" ] && $extcheck || cat;} +} +specopt() { + cmd=`echo $cmd | sed 's/%//'` # strip leading '%' + echo ".`echo $cmd | tr '[a-z]' '[A-Z]'` $rest" # show what we got + specopt_$cmd # do what they asked for +} + +# +# wait for background processes to finish +# and print their output +# NB: We wait for processes in a FIFO order +# The order in which they finish does not matter +# +waitforbgprocs() { + while [ "$bgprocs" ]; do + set $bgprocs + proc=$1 # get the first one + shift 1 # remove it from the list + bgprocs="$@" + IFS=":" + set $proc # split into lineno,pid + testline=$1 jobnum=$2 pid=$3 + unset IFS + + while kill -0 $pid 2>/dev/null; do + sleep 1 + done + wait $pid # capture the exit code + + echo ".BG test line $testline/job $jobnum finished (exit code: $?):" + echo "==========test:$testline:$jobnum start output==========" + cat $OUTDIR/bg$$-$testline-$jobnum + echo "==========test:$testline:$jobnum end output==========" + rm -f $OUTDIR/bg$$-$testline-$jobnum + done +} + +# +# substitute variables in the test line +# +substvars() { + sed " + s/%t/$test_cnt/g + s/%l/$line/g + s/%j/$job_cnt/g + s/%i/$repeat_cnt/g + " +} + +dotest() { + echo -n "." >&3 + test_cnt=$(($test_cnt+1)) + describe_$cmd # show what we are about to do + lrm_$cmd | # and execute the command + { [ "$extcheck" ] && $extcheck || cat;} +} +runonetest() { + eval `echo $rest | substvars` # set parameters + if [ "$job_cnt" -le "$bgprocs_num" ]; then + echo .BG test line $line/job $job_cnt runs in background + dotest > $OUTDIR/bg$$-$line-$job_cnt 2>&1 & + bgprocs="$bgprocs $line:$job_cnt:$!" + job_cnt=$(($job_cnt+1)) + else + dotest + fi +} +runtest() { + while [ $repeat_cnt -le $repeat_limit ]; do + runonetest + resetvars # unset all variables + repeat_cnt=$(($repeat_cnt+1)) + done + repeat_limit=1 repeat_cnt=1 +} + +# +# run the tests +# +bgprocs_num=0 job_cnt=1 +repeat_limit=1 repeat_cnt=1 +line=1 +test_cnt=1 + +while read cmd rest; do + case "$cmd" in + "") : empty ;; + "#"*) : a comment ;; + "%stop") break ;; + "%"*) specopt ;; + *) runtest ;; + esac + line=$(($line+1)) +done +waitforbgprocs diff --git a/lrm/test/language b/lrm/test/language new file mode 100644 index 0000000..d2785e8 --- /dev/null +++ b/lrm/test/language @@ -0,0 +1,16 @@ +The meta language and how it translates to the lrmadmin options: + +list:-L +add:-A %r %C %T %P +del:-D %r +flush:-F %r +state:-S %r +info:-I %r +exec:-E %r %o %t %i %e + +classes:-C +types:-T %C +classmeta:-O %C +meta:-M %C %T %P +provider:-P %C %T + diff --git a/lrm/test/lrmadmin-interface b/lrm/test/lrmadmin-interface new file mode 100644 index 0000000..4eb1656 --- /dev/null +++ b/lrm/test/lrmadmin-interface @@ -0,0 +1,43 @@ +lrm_list() { + $LRMADMIN -L +} +lrm_add() { + $LRMADMIN -A ${rsc:-$dflt_rsc} \ + ${class:-$dflt_class} ${type:-$dflt_type} \ + ${provider:-$dflt_provider} \ + $args +} +lrm_del() { + $LRMADMIN -D ${rsc:-$dflt_rsc} +} +lrm_flush() { + $LRMADMIN -F ${rsc:-$dflt_rsc} +} +lrm_state() { + $LRMADMIN -S ${rsc:-$dflt_rsc} +} +lrm_info() { + $LRMADMIN -I ${rsc:-$dflt_rsc} +} +lrm_exec() { + $LRMADMIN -E ${rsc:-$dflt_rsc} \ + ${operation:-$dflt_operation} \ + ${timeout:-$dflt_timeout} ${interval:-$dflt_interval} \ + ${targetrc:-$dflt_targetrc} $args +} + +lrm_classes() { + $LRMADMIN -C +} +lrm_types() { + $LRMADMIN -T ${class:-$dflt_class} +} +lrm_classmeta() { + $LRMADMIN -O ${class:-$dflt_class} +} +lrm_meta() { + $LRMADMIN -M ${class:-$dflt_class} ${type:-$dflt_type} ${provider:-$dflt_provider} +} +lrm_provider() { + $LRMADMIN -P ${class:-$dflt_class} ${type:-$dflt_type} +} diff --git a/lrm/test/lrmregtest-lsb b/lrm/test/lrmregtest-lsb new file mode 100644 index 0000000..4692b17 --- /dev/null +++ b/lrm/test/lrmregtest-lsb @@ -0,0 +1,54 @@ +#!/bin/sh +# +# WARNING: This script is for LRM regressions tests only +# +### BEGIN INIT INFO +# Provides: lrmregtest +# Required-Start: +# Should-Start: +# Required-Stop: +# Should-Stop: +# Default-Start: +# Default-Stop: +# Short-Description: LRM regression tests LSB RA +# Description: LRM regression tests LSB RA +### END INIT INFO + +TYPE=lrmregtest +# depends on resource-agents and the OCF +: ${OCF_ROOT:=/usr/lib/ocf} +. ${OCF_ROOT}/lib/heartbeat/ocf-shellfuncs + +case "$1" in + start) + echo -n "Starting $TYPE" + ha_pseudo_resource lrmregtest_lsb start + ;; + stop) + echo -n "Shutting down $TYPE" + ha_pseudo_resource lrmregtest_lsb stop + ;; + status) + echo -n "Checking for $TYPE" + ha_pseudo_resource lrmregtest_lsb monitor + if [ $? -eq 0 ]; then + echo " running" + exit 0 + else + echo " stopped" + exit 3 + fi + ;; + *) + echo "Usage: $0 {start|stop|status}" + exit 1 + ;; +esac + +if [ $? -eq 0 ]; then + echo " OK" + exit 0 +else + echo " failed" + exit 1 +fi diff --git a/lrm/test/lrmregtest.in b/lrm/test/lrmregtest.in new file mode 100644 index 0000000..001a662 --- /dev/null +++ b/lrm/test/lrmregtest.in @@ -0,0 +1,220 @@ +#!/bin/sh +# +# +# lrmregtest OCF RA. Does nothing but wait a few seconds, can be +# configured to fail occassionally. +# +# updated to support the LRM regression testing. +# +# Copyright (c) 2007 SUSE LINUX AG, Dejan Muhamedagic +# All Rights Reserved. +# +# Copyright (c) 2004 SUSE LINUX AG, Lars Marowsky-Brée +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# + +####################################################################### +# Initialization: + +. ${OCF_ROOT}/lib/heartbeat/ocf-shellfuncs + +####################################################################### + +meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="lrmregtest" version="0.9"> +<version>1.0</version> + +<longdesc lang="en"> +This is a lrmregtest Resource Agent. Use for LRM regression +testing. +</longdesc> +<shortdesc lang="en">lrmregtest resource agent</shortdesc> + +<parameters> +<parameter name="delay" unique="0"> +<longdesc lang="en"> +How long to delay before each action. +</longdesc> +<shortdesc lang="en">Action delay</shortdesc> +<content type="integer" default="0" /> +</parameter> + +<parameter name="check_parallel" unique="0"> +<longdesc lang="en"> +Complain loudly if they try to run us in parallel on the same resource. +</longdesc> +<shortdesc lang="en">Report error if run twice at the same time</shortdesc> +<content type="boolean" default="true" /> +</parameter> + +<parameter name="ignore_TERM" unique="0"> +<longdesc lang="en"> +Process the TERM signal and don't exit. +</longdesc> +<shortdesc lang="en">No TERM ain't gonna kill us.</shortdesc> +<content type="boolean" default="false" /> +</parameter> + +<parameter name="verbose" unique="0"> +<longdesc lang="en"> +Print more information. +</longdesc> +<shortdesc lang="en">Be verbose.</shortdesc> +<content type="boolean" default="false" /> +</parameter> + +</parameters> + +<actions> +<action name="start" timeout="90" /> +<action name="stop" timeout="100" /> +<action name="monitor" timeout="20" interval="10" depth="0" start-delay="0" /> +<action name="reload" timeout="90" /> +<action name="migrate_to" timeout="100" /> +<action name="migrate_from" timeout="90" /> +<action name="meta-data" timeout="5" /> +<action name="validate-all" timeout="30" /> +</actions> +</resource-agent> +END +} + +####################################################################### + +# don't exit on TERM, to test that lrmd makes sure that we do exit +sigterm_handler() { + ocf_log info "They use TERM to bring us down. No such luck." + return +} + +dummy_usage() { + cat <<END +usage: $0 {start|stop|monitor|migrate_to|migrate_from|validate-all|meta-data} + +Expects to have a fully populated OCF RA-compliant environment set. +END +} + +# signals interrupt slow calls (sleep) +# this is an approximation (after all it's just a dummy) +sleepsleep() { + delay=$1 + now=`perl -e 'print time()'` + by=$(($now+$delay)) + while [ $now -lt $by ]; do + ocf_log debug "Gonna sleep for $(($by-$now)) seconds..." + sleep $(($by-$now)) + now=`perl -e 'print time()'` + done +} +dummy_start() { + sleepsleep $OCF_RESKEY_delay + ha_pseudo_resource lrmregtest_${OCF_RESOURCE_INSTANCE} start +} + +dummy_stop() { + sleepsleep $OCF_RESKEY_delay + ha_pseudo_resource lrmregtest_${OCF_RESOURCE_INSTANCE} stop +} + +dummy_monitor() { + sleepsleep $OCF_RESKEY_delay + ha_pseudo_resource lrmregtest_${OCF_RESOURCE_INSTANCE} monitor +} + +dummy_validate() { + exit $OC_ERR_UNIMPLEMENTED +} + +verbose() { + [ "$OCF_RESKEY_verbose" != 0 ] +} +environment() { + echo "OCF environment variables:" + set | egrep 'OCF_RESKEY|OCF_RESOURCE_INSTANCE' +} +invocation() { + echo "invoked with args: $@" +} + +: ${OCF_RESKEY_delay=0} +: ${OCF_RESKEY_check_parallel=1} +: ${OCF_RESKEY_verbose=0} +: ${OCF_RESKEY_ignore_TERM=0} + +verbose && environment + +lockf=` + ha_pseudo_resource lrmregtest_${OCF_RESOURCE_INSTANCE} print | + sed 's/$/.lock/' +` + +check4parallel() { + if [ -f "$lockf" ] && kill -0 `cat $lockf` 2>/dev/null + then + ocf_log err "There is another instance of ${OCF_RESOURCE_INSTANCE} running: pid `cat $lockf`." + exit $OCF_ERR_GENERIC + fi +} + +[ "$OCF_RESKEY_check_parallel" = 1 ] && + check4parallel + +[ "$OCF_RESKEY_ignore_TERM" = 1 ] && + trap sigterm_handler TERM + +echo $$ > $lockf +trap "rm -f $lockf" EXIT + +verbose && invocation $@ + +case $__OCF_ACTION in +meta-data) meta_data + exit $OCF_SUCCESS + ;; +start) dummy_start;; +stop) dummy_stop;; +monitor) dummy_monitor;; +migrate_to) ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} to ${OCF_RESKEY_CRM_meta_migrate_to}." + dummy_stop + ;; +migrate_from) ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} to ${OCF_RESKEY_CRM_meta_migrated_from}." + dummy_start + ;; +reload) ocf_log err "Reloading..." + dummy_start + ;; +validate-all) dummy_validate;; +usage|help) dummy_usage + exit $OCF_SUCCESS + ;; +*) dummy_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac +rc=$? +ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" +exit $rc + diff --git a/lrm/test/plugintest.c b/lrm/test/plugintest.c new file mode 100644 index 0000000..d25c46d --- /dev/null +++ b/lrm/test/plugintest.c @@ -0,0 +1,84 @@ +/* File: plugintest.c + * Description: A small,simple tool to test RA execution plugin + * + * Author: Sun Jiang Dong <sunjd@cn.ibm.com> + * Copyright (c) 2004 International Business Machines + * + * Todo: security verification + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#include <glib.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <pils/plugin.h> +#include <pils/generic.h> +#include <lrm/raexec.h> + +static void +g_print_item(gpointer data, gpointer user_data) +{ + printf("%s\n", (char*)data); +} + + +int main(void) +{ + PILPluginUniv * PluginLoadingSystem = NULL; + GHashTable * RAExecFuncs = NULL; + GList * ratype_list; + struct RAExecOps * RAExec; + /* + GHashTable * cmd_params; + */ + int ret; + + PILGenericIfMgmtRqst RegisterRqsts[]= { + {"RAExec", &RAExecFuncs, NULL, NULL, NULL}, + { NULL, NULL, NULL, NULL, NULL} }; + + PluginLoadingSystem = NewPILPluginUniv ("/usr/lib/heartbeat/plugins"); + + PILLoadPlugin(PluginLoadingSystem , "InterfaceMgr", "generic" , &RegisterRqsts); + + PILLoadPlugin(PluginLoadingSystem , "RAExec", "ocf", NULL); + RAExec = g_hash_table_lookup(RAExecFuncs,"ocf"); + ret = RAExec->get_resource_list(&ratype_list); + printf("length=%d\n", g_list_length(ratype_list)); + if (ret >= 0) { + g_list_foreach(ratype_list, g_print_item, NULL); + } + + /* + PILLoadPlugin(PluginLoadingSystem , "RAExec", "lsb", NULL); + RAExec = g_hash_table_lookup(RAExecFuncs,"lsb"); + cmd_params = g_hash_table_new(g_str_hash, g_str_equal); + g_hash_table_insert(cmd_params, g_strdup("1"), g_strdup("par1")); + g_hash_table_insert(cmd_params, g_strdup("2"), g_strdup("par2")); + ret = RAExec->execra("/tmp/test.sh", "start", cmd_params,NULL); + */ + + /* For test the dealing with directory appended to RA */ + /* + PILLoadPlugin(PluginLoadingSystem , "RAExec", "ocf", NULL); + RAExec = g_hash_table_lookup(RAExecFuncs,"ocf"); + if (0>RAExec->execra("/root/linux-ha-checkout/linux-ha/lrm/test.sh", + "stop",NULL,NULL, TRUE, &key)) + */ + printf("execra result: ret = %d\n", ret); + return -1; +} diff --git a/lrm/test/regression.sh.in b/lrm/test/regression.sh.in new file mode 100755 index 0000000..550321e --- /dev/null +++ b/lrm/test/regression.sh.in @@ -0,0 +1,248 @@ +#!/bin/sh + + # Copyright (C) 2007 Dejan Muhamedagic <dmuhamedagic@suse.de> + # + # This program is free software; you can redistribute it and/or + # modify it under the terms of the GNU General Public + # License as published by the Free Software Foundation; either + # version 2.1 of the License, or (at your option) any later version. + # + # This software is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + # General Public License for more details. + # + # You should have received a copy of the GNU General Public + # License along with this library; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + # + +OCF_ROOT=@OCF_ROOT_DIR@ +export OCF_ROOT +if [ -z "$OCF_ROOT" ]; then + [ -d /usr/lib/ocf ] && OCF_ROOT=/usr/lib/ocf +fi +if [ ! -d "$OCF_ROOT" ]; then + echo "OCF_ROOT environment variable not set" + exit 2 +fi + +TESTDIR=${TESTDIR:-testcases} +DFLT_TESTSET=basicset +OUTDIR=${OUTDIR:-output} +LRMD_OUTF="$OUTDIR/lrmd.out" +LRMD_LOGF="$OUTDIR/lrmd.log" +LRMD_DEBUGF="$OUTDIR/lrmd.debug" +OUTF="$OUTDIR/regression.out" +LRMADMIN="@sbindir@/lrmadmin" +LRMD_OPTS="-vvv" +STONITHD_OPTS="-at" +DIFF_OPTS="--ignore-all-space -U 1" +common_filter=$TESTDIR/common.filter +common_exclf=$TESTDIR/common.excl +OCF_RA=$OCF_ROOT/resource.d/heartbeat/lrmregtest +LSB_RA=@LSB_RA_DIR@/lrmregtest +export OUTDIR TESTDIR LRMADMIN + +logmsg() { + echo "`date`: $*" | tee -a $LRMD_DEBUGF | tee -a $LRMD_LOGF +} +abspath() { + echo $1 | grep -qs "^/" && + echo $1 || + echo `pwd`/$1 +} + +usage() { + cat<<EOF + +usage: $0 [-q] [testcase...|set:testset] + +Test lrmd using supplied testcases. If none are given, +set:basicset is used. All testcases and sets are in testcases/. +See also README.regression for description. + +-q: quiet operation (no progress shown) + +EOF +exit 2 +} + +if [ `id -u` != 0 ]; then + echo "sorry, but i talk to root only" + exit 2 +fi +cd `dirname $0` +if [ ! -d "$TESTDIR" ]; then + echo "$0: $TESTDIR does not exit" + usage +fi + +which xmllint >/dev/null 2>&1 || { + echo "WARNING: xmllint not available, some of the tests may fail" +} + +rm -f $LRMD_LOGF $LRMD_DEBUGF + +# make lrmd log to our files only +HA_logfile=`abspath $LRMD_LOGF` +HA_debugfile=`abspath $LRMD_DEBUGF` +HA_use_logd=no +HA_logfacility="" +export HA_logfile HA_debugfile HA_use_logd HA_logfacility + +mkdir -p $OUTDIR +. ${OCF_ROOT}/lib/heartbeat/ocf-shellfuncs + +args=`getopt hq $*` +[ $? -ne 0 ] && usage +eval set -- "$args" + +SILENT="" +while [ x"$1" != x ]; do + case "$1" in + -h) usage;; + -q) SILENT=1;; + --) shift 1; break;; + *) usage;; + esac + shift 1 +done + +exec >$OUTF 2>&1 +if [ "$SILENT" = 1 ]; then + exec 3>/dev/null +else + exec 3>/dev/tty +fi + +start_stonithd() { + echo "starting stonithd" >&3 + $HA_BIN/stonithd -s 2>/dev/null + if [ $? -ne 0 ]; then + STOP_STONITHD=1 + $HA_BIN/stonithd $STONITHD_OPTS + sleep 1 + $HA_BIN/stonithd -s 2>/dev/null + else + STOP_STONITHD= + fi +} +stop_stonithd() { + if [ "$STOP_STONITHD" ]; then + echo "stopping stonithd" >&3 + $HA_BIN/stonithd -k >/dev/null 2>&1 + fi +} +start_lrmd() { + echo "starting lrmd" >&3 + $HA_BIN/lrmd -s 2>/dev/null + if [ $? -eq 3 ]; then + #strace -o /tmp/lrmd.trc $HA_BIN/lrmd $LRMD_OPTS >$LRMD_OUTF 2>&1 & + $HA_BIN/lrmd $LRMD_OPTS >$LRMD_OUTF 2>&1 & + sleep 1 + $HA_BIN/lrmd -s 2>/dev/null + else + echo "lrmd already running; can't proceed" >&3 + return 2 + fi +} +stop_lrmd() { + echo "stopping lrmd" >&3 + $HA_BIN/lrmd -k +} +cp_ra() { + cp -p lrmregtest $OCF_RA + chmod +x $OCF_RA + cp -p lrmregtest-lsb $LSB_RA + chmod +x $LSB_RA +} +rm_ra() { + rm -f $OCF_RA $LSB_RA +} + +cp_ra +start_lrmd || exit $? +# start_stonithd || exit $? +trap "stop_lrmd; stop_stonithd; rm_ra" EXIT + +setenvironment() { + filterf=$TESTDIR/$testcase.filter + exclf=$TESTDIR/$testcase.excl + log_filter=$TESTDIR/$testcase.log_filter + expf=$TESTDIR/$testcase.exp + outf=$OUTDIR/$testcase.out + difff=$OUTDIR/$testcase.diff +} + +filter_output() { + { [ -x $common_filter ] && $common_filter || cat;} | + { [ -f $common_exclf ] && egrep -vf $common_exclf || cat;} | + { [ -x $filterf ] && $filterf || cat;} | + { [ -f $exclf ] && egrep -vf $exclf || cat;} +} + +dumpcase() { + cat<<EOF +---------- +testcase $testcase failed +output is in $outf +diff (from $difff): +`cat $difff` +---------- +EOF +} + +runtestcase() { + setenvironment + echo -n "$testcase" >&3 + logmsg "BEGIN testcase $testcase" + ./evaltest.sh < $TESTDIR/$testcase > $outf 2>&1 + + filter_output < $outf | + if [ "$prepare" ]; then + echo " saving to expect file" >&3 + cat > $expf + else + echo -n " checking..." >&3 + diff $DIFF_OPTS $expf - > $difff + if [ $? -ne 0 ]; then + echo " FAIL" >&3 + dumpcase + return 1 + else + echo " PASS" >&3 + rm -f $outf $difff + fi + fi + sed -n "/BEGIN testcase $testcase/,\$p" $LRMD_LOGF | + { [ -x $log_filter ] && $log_filter || cat;} | + egrep '(CRIT|ERROR):' + logmsg "END testcase $testcase" +} + +[ "$1" = prepare ] && { prepare=1; shift 1;} +[ $# -eq 0 ] && set "set:$DFLT_TESTSET" + +for a; do + if [ "$a" -a -f "$TESTDIR/$a" ]; then + testcase=$a + runtestcase + else + echo "$a" | grep -q "^set:" && + TESTSET=$TESTDIR/`echo $a | sed 's/set://'` + while read testcase; do + runtestcase + done < $TESTSET + fi +done + +if egrep -wv '(BEGIN|END) testcase' $OUTF >/dev/null +then + echo "seems like some tests failed or else something not expected" + echo "check $OUTF and diff files in $OUTDIR" + echo "in case you wonder what lrmd was doing, read $LRMD_LOGF and $LRMD_DEBUGF" + exit 1 +else + rm -f $OUTF $LRMD_OUTF +fi >&3 diff --git a/lrm/test/testcases/BSC b/lrm/test/testcases/BSC new file mode 100644 index 0000000..157fb6c --- /dev/null +++ b/lrm/test/testcases/BSC @@ -0,0 +1,4 @@ +rscmgmt +metadata +rscexec +stonith diff --git a/lrm/test/testcases/Makefile.am b/lrm/test/testcases/Makefile.am new file mode 100644 index 0000000..49728d9 --- /dev/null +++ b/lrm/test/testcases/Makefile.am @@ -0,0 +1,27 @@ +# +# Author: Sun Jiang Dong <sunjd@cn.ibm.com> +# Copyright (c) 2004 International Business Machines +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +testcasesdir = $(datadir)/$(PACKAGE_NAME)/lrmtest/testcases +testcases_SCRIPTS = common.filter ra-list.sh rscmgmt.log_filter xmllint.sh +testcases_DATA = BSC basicset metadata metadata.exp rscexec \ + rscexec.exp rscmgmt rscmgmt.exp \ + stonith stonith.exp +# shouldn't need this next line... +EXTRA_DIST = $(testcases_SCRIPTS) $(testcases_DATA) diff --git a/lrm/test/testcases/basicset b/lrm/test/testcases/basicset new file mode 100644 index 0000000..62b9c04 --- /dev/null +++ b/lrm/test/testcases/basicset @@ -0,0 +1,6 @@ +rscmgmt +metadata +rscexec +stonith +serialize +flood diff --git a/lrm/test/testcases/common.filter b/lrm/test/testcases/common.filter new file mode 100755 index 0000000..f95e9d8 --- /dev/null +++ b/lrm/test/testcases/common.filter @@ -0,0 +1,27 @@ +#!/bin/sh + +sed ' +/^lrmadmin/s/([0-9][0-9]*)/()/ +/^lrmadmin/s/^lrmadmin[^:]*: [^ ]* // +s/call_id=[0-9][0-9]*/call_id=(removed)/ +/run at:/d +/last rc change at:/d +/queue time:/d +' | +awk ' +/Waiting for lrmd to callback.../ { n=1; next; } +n==1 && /----------------operation--------------/ { n++; next; } +n==2 && /type:/ { op=$0; sub("type:","",op); next } +n==2 && /operation status:/ { desc=$0; sub("operation status:","",desc); next } +n==2 && /op_status:/ { stat=$0; sub("op_status: *","",stat); next } +n==2 && /return code:/ { rc=$0; sub("return code: *","",rc); next } +n==2 && /output data:/ { n++; next; } +n==3 && /---------------------------------------/ { + printf("> %s %s (status=%s,rc=%s): %s\n",op,desc,stat,rc,substr(output,2)); + n=0; + output=""; + next; +} +n==3 && $1!="" { output=output"/"$0; next; } +{ print } +' diff --git a/lrm/test/testcases/flood b/lrm/test/testcases/flood new file mode 100644 index 0000000..de6d742 --- /dev/null +++ b/lrm/test/testcases/flood @@ -0,0 +1,19 @@ +# 30 secs should be enough even on slow machines +list +%setenv dflt_timeout=30000 +# add 64 resources +%repeat 64 +add rsc=r%i args="delay=0" +# start all in background +%bgrepeat 64 +exec rsc=r%i operation=start +%sleep 1 +# and run a monitor on all in background +%bgrepeat 64 +exec rsc=r%i operation=monitor +%sleep 1 +# finally, stop all +%repeat 64 +exec rsc=r%i operation=stop +%repeat 64 +del rsc=r%i diff --git a/lrm/test/testcases/flood.exp b/lrm/test/testcases/flood.exp new file mode 100644 index 0000000..cf8a2bb --- /dev/null +++ b/lrm/test/testcases/flood.exp @@ -0,0 +1,1354 @@ +.TRY List resources +Currently no resources are managed by LRM. +.SETENV dflt_timeout=30000 +.REPEAT 64 +.TRY Add resource r1 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r2 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r3 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r4 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r5 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r6 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r7 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r8 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r9 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r10 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r11 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r12 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r13 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r14 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r15 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r16 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r17 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r18 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r19 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r20 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r21 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r22 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r23 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r24 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r25 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r26 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r27 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r28 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r29 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r30 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r31 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r32 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r33 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r34 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r35 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r36 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r37 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r38 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r39 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r40 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r41 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r42 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r43 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r44 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r45 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r46 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r47 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r48 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r49 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r50 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r51 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r52 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r53 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r54 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r55 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r56 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r57 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r58 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r59 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r60 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r61 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r62 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r63 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY Add resource r64 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.BGREPEAT 64 +.BG test line 9/job 1 runs in background +.BG test line 9/job 2 runs in background +.BG test line 9/job 3 runs in background +.BG test line 9/job 4 runs in background +.BG test line 9/job 5 runs in background +.BG test line 9/job 6 runs in background +.BG test line 9/job 7 runs in background +.BG test line 9/job 8 runs in background +.BG test line 9/job 9 runs in background +.BG test line 9/job 10 runs in background +.BG test line 9/job 11 runs in background +.BG test line 9/job 12 runs in background +.BG test line 9/job 13 runs in background +.BG test line 9/job 14 runs in background +.BG test line 9/job 15 runs in background +.BG test line 9/job 16 runs in background +.BG test line 9/job 17 runs in background +.BG test line 9/job 18 runs in background +.BG test line 9/job 19 runs in background +.BG test line 9/job 20 runs in background +.BG test line 9/job 21 runs in background +.BG test line 9/job 22 runs in background +.BG test line 9/job 23 runs in background +.BG test line 9/job 24 runs in background +.BG test line 9/job 25 runs in background +.BG test line 9/job 26 runs in background +.BG test line 9/job 27 runs in background +.BG test line 9/job 28 runs in background +.BG test line 9/job 29 runs in background +.BG test line 9/job 30 runs in background +.BG test line 9/job 31 runs in background +.BG test line 9/job 32 runs in background +.BG test line 9/job 33 runs in background +.BG test line 9/job 34 runs in background +.BG test line 9/job 35 runs in background +.BG test line 9/job 36 runs in background +.BG test line 9/job 37 runs in background +.BG test line 9/job 38 runs in background +.BG test line 9/job 39 runs in background +.BG test line 9/job 40 runs in background +.BG test line 9/job 41 runs in background +.BG test line 9/job 42 runs in background +.BG test line 9/job 43 runs in background +.BG test line 9/job 44 runs in background +.BG test line 9/job 45 runs in background +.BG test line 9/job 46 runs in background +.BG test line 9/job 47 runs in background +.BG test line 9/job 48 runs in background +.BG test line 9/job 49 runs in background +.BG test line 9/job 50 runs in background +.BG test line 9/job 51 runs in background +.BG test line 9/job 52 runs in background +.BG test line 9/job 53 runs in background +.BG test line 9/job 54 runs in background +.BG test line 9/job 55 runs in background +.BG test line 9/job 56 runs in background +.BG test line 9/job 57 runs in background +.BG test line 9/job 58 runs in background +.BG test line 9/job 59 runs in background +.BG test line 9/job 60 runs in background +.BG test line 9/job 61 runs in background +.BG test line 9/job 62 runs in background +.BG test line 9/job 63 runs in background +.BG test line 9/job 64 runs in background +.SLEEP 1 +.BGREPEAT 64 +.BG test line 13/job 1 runs in background +.BG test line 13/job 2 runs in background +.BG test line 13/job 3 runs in background +.BG test line 13/job 4 runs in background +.BG test line 13/job 5 runs in background +.BG test line 13/job 6 runs in background +.BG test line 13/job 7 runs in background +.BG test line 13/job 8 runs in background +.BG test line 13/job 9 runs in background +.BG test line 13/job 10 runs in background +.BG test line 13/job 11 runs in background +.BG test line 13/job 12 runs in background +.BG test line 13/job 13 runs in background +.BG test line 13/job 14 runs in background +.BG test line 13/job 15 runs in background +.BG test line 13/job 16 runs in background +.BG test line 13/job 17 runs in background +.BG test line 13/job 18 runs in background +.BG test line 13/job 19 runs in background +.BG test line 13/job 20 runs in background +.BG test line 13/job 21 runs in background +.BG test line 13/job 22 runs in background +.BG test line 13/job 23 runs in background +.BG test line 13/job 24 runs in background +.BG test line 13/job 25 runs in background +.BG test line 13/job 26 runs in background +.BG test line 13/job 27 runs in background +.BG test line 13/job 28 runs in background +.BG test line 13/job 29 runs in background +.BG test line 13/job 30 runs in background +.BG test line 13/job 31 runs in background +.BG test line 13/job 32 runs in background +.BG test line 13/job 33 runs in background +.BG test line 13/job 34 runs in background +.BG test line 13/job 35 runs in background +.BG test line 13/job 36 runs in background +.BG test line 13/job 37 runs in background +.BG test line 13/job 38 runs in background +.BG test line 13/job 39 runs in background +.BG test line 13/job 40 runs in background +.BG test line 13/job 41 runs in background +.BG test line 13/job 42 runs in background +.BG test line 13/job 43 runs in background +.BG test line 13/job 44 runs in background +.BG test line 13/job 45 runs in background +.BG test line 13/job 46 runs in background +.BG test line 13/job 47 runs in background +.BG test line 13/job 48 runs in background +.BG test line 13/job 49 runs in background +.BG test line 13/job 50 runs in background +.BG test line 13/job 51 runs in background +.BG test line 13/job 52 runs in background +.BG test line 13/job 53 runs in background +.BG test line 13/job 54 runs in background +.BG test line 13/job 55 runs in background +.BG test line 13/job 56 runs in background +.BG test line 13/job 57 runs in background +.BG test line 13/job 58 runs in background +.BG test line 13/job 59 runs in background +.BG test line 13/job 60 runs in background +.BG test line 13/job 61 runs in background +.BG test line 13/job 62 runs in background +.BG test line 13/job 63 runs in background +.BG test line 13/job 64 runs in background +.SLEEP 1 +.REPEAT 64 +.TRY Exec r1 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r2 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r3 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r4 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r5 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r6 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r7 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r8 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r9 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r10 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r11 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r12 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r13 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r14 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r15 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r16 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r17 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r18 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r19 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r20 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r21 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r22 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r23 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r24 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r25 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r26 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r27 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r28 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r29 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r30 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r31 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r32 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r33 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r34 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r35 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r36 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r37 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r38 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r39 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r40 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r41 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r42 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r43 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r44 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r45 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r46 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r47 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r48 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r49 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r50 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r51 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r52 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r53 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r54 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r55 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r56 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r57 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r58 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r59 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r60 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r61 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r62 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r63 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec r64 op=stop timeout=30000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.REPEAT 64 +.TRY Delete resource r1 +Succeeded in deleting this resource. +.TRY Delete resource r2 +Succeeded in deleting this resource. +.TRY Delete resource r3 +Succeeded in deleting this resource. +.TRY Delete resource r4 +Succeeded in deleting this resource. +.TRY Delete resource r5 +Succeeded in deleting this resource. +.TRY Delete resource r6 +Succeeded in deleting this resource. +.TRY Delete resource r7 +Succeeded in deleting this resource. +.TRY Delete resource r8 +Succeeded in deleting this resource. +.TRY Delete resource r9 +Succeeded in deleting this resource. +.TRY Delete resource r10 +Succeeded in deleting this resource. +.TRY Delete resource r11 +Succeeded in deleting this resource. +.TRY Delete resource r12 +Succeeded in deleting this resource. +.TRY Delete resource r13 +Succeeded in deleting this resource. +.TRY Delete resource r14 +Succeeded in deleting this resource. +.TRY Delete resource r15 +Succeeded in deleting this resource. +.TRY Delete resource r16 +Succeeded in deleting this resource. +.TRY Delete resource r17 +Succeeded in deleting this resource. +.TRY Delete resource r18 +Succeeded in deleting this resource. +.TRY Delete resource r19 +Succeeded in deleting this resource. +.TRY Delete resource r20 +Succeeded in deleting this resource. +.TRY Delete resource r21 +Succeeded in deleting this resource. +.TRY Delete resource r22 +Succeeded in deleting this resource. +.TRY Delete resource r23 +Succeeded in deleting this resource. +.TRY Delete resource r24 +Succeeded in deleting this resource. +.TRY Delete resource r25 +Succeeded in deleting this resource. +.TRY Delete resource r26 +Succeeded in deleting this resource. +.TRY Delete resource r27 +Succeeded in deleting this resource. +.TRY Delete resource r28 +Succeeded in deleting this resource. +.TRY Delete resource r29 +Succeeded in deleting this resource. +.TRY Delete resource r30 +Succeeded in deleting this resource. +.TRY Delete resource r31 +Succeeded in deleting this resource. +.TRY Delete resource r32 +Succeeded in deleting this resource. +.TRY Delete resource r33 +Succeeded in deleting this resource. +.TRY Delete resource r34 +Succeeded in deleting this resource. +.TRY Delete resource r35 +Succeeded in deleting this resource. +.TRY Delete resource r36 +Succeeded in deleting this resource. +.TRY Delete resource r37 +Succeeded in deleting this resource. +.TRY Delete resource r38 +Succeeded in deleting this resource. +.TRY Delete resource r39 +Succeeded in deleting this resource. +.TRY Delete resource r40 +Succeeded in deleting this resource. +.TRY Delete resource r41 +Succeeded in deleting this resource. +.TRY Delete resource r42 +Succeeded in deleting this resource. +.TRY Delete resource r43 +Succeeded in deleting this resource. +.TRY Delete resource r44 +Succeeded in deleting this resource. +.TRY Delete resource r45 +Succeeded in deleting this resource. +.TRY Delete resource r46 +Succeeded in deleting this resource. +.TRY Delete resource r47 +Succeeded in deleting this resource. +.TRY Delete resource r48 +Succeeded in deleting this resource. +.TRY Delete resource r49 +Succeeded in deleting this resource. +.TRY Delete resource r50 +Succeeded in deleting this resource. +.TRY Delete resource r51 +Succeeded in deleting this resource. +.TRY Delete resource r52 +Succeeded in deleting this resource. +.TRY Delete resource r53 +Succeeded in deleting this resource. +.TRY Delete resource r54 +Succeeded in deleting this resource. +.TRY Delete resource r55 +Succeeded in deleting this resource. +.TRY Delete resource r56 +Succeeded in deleting this resource. +.TRY Delete resource r57 +Succeeded in deleting this resource. +.TRY Delete resource r58 +Succeeded in deleting this resource. +.TRY Delete resource r59 +Succeeded in deleting this resource. +.TRY Delete resource r60 +Succeeded in deleting this resource. +.TRY Delete resource r61 +Succeeded in deleting this resource. +.TRY Delete resource r62 +Succeeded in deleting this resource. +.TRY Delete resource r63 +Succeeded in deleting this resource. +.TRY Delete resource r64 +Succeeded in deleting this resource. +.BG test line 9/job 1 finished (exit code: 0): +==========test:9:1 start output========== +.TRY Exec r1 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:1 end output========== +.BG test line 9/job 2 finished (exit code: 0): +==========test:9:2 start output========== +.TRY Exec r2 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:2 end output========== +.BG test line 9/job 3 finished (exit code: 0): +==========test:9:3 start output========== +.TRY Exec r3 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:3 end output========== +.BG test line 9/job 4 finished (exit code: 0): +==========test:9:4 start output========== +.TRY Exec r4 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:4 end output========== +.BG test line 9/job 5 finished (exit code: 0): +==========test:9:5 start output========== +.TRY Exec r5 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:5 end output========== +.BG test line 9/job 6 finished (exit code: 0): +==========test:9:6 start output========== +.TRY Exec r6 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:6 end output========== +.BG test line 9/job 7 finished (exit code: 0): +==========test:9:7 start output========== +.TRY Exec r7 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:7 end output========== +.BG test line 9/job 8 finished (exit code: 0): +==========test:9:8 start output========== +.TRY Exec r8 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:8 end output========== +.BG test line 9/job 9 finished (exit code: 0): +==========test:9:9 start output========== +.TRY Exec r9 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:9 end output========== +.BG test line 9/job 10 finished (exit code: 0): +==========test:9:10 start output========== +.TRY Exec r10 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:10 end output========== +.BG test line 9/job 11 finished (exit code: 0): +==========test:9:11 start output========== +.TRY Exec r11 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:11 end output========== +.BG test line 9/job 12 finished (exit code: 0): +==========test:9:12 start output========== +.TRY Exec r12 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:12 end output========== +.BG test line 9/job 13 finished (exit code: 0): +==========test:9:13 start output========== +.TRY Exec r13 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:13 end output========== +.BG test line 9/job 14 finished (exit code: 0): +==========test:9:14 start output========== +.TRY Exec r14 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:14 end output========== +.BG test line 9/job 15 finished (exit code: 0): +==========test:9:15 start output========== +.TRY Exec r15 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:15 end output========== +.BG test line 9/job 16 finished (exit code: 0): +==========test:9:16 start output========== +.TRY Exec r16 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:16 end output========== +.BG test line 9/job 17 finished (exit code: 0): +==========test:9:17 start output========== +.TRY Exec r17 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:17 end output========== +.BG test line 9/job 18 finished (exit code: 0): +==========test:9:18 start output========== +.TRY Exec r18 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:18 end output========== +.BG test line 9/job 19 finished (exit code: 0): +==========test:9:19 start output========== +.TRY Exec r19 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:19 end output========== +.BG test line 9/job 20 finished (exit code: 0): +==========test:9:20 start output========== +.TRY Exec r20 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:20 end output========== +.BG test line 9/job 21 finished (exit code: 0): +==========test:9:21 start output========== +.TRY Exec r21 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:21 end output========== +.BG test line 9/job 22 finished (exit code: 0): +==========test:9:22 start output========== +.TRY Exec r22 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:22 end output========== +.BG test line 9/job 23 finished (exit code: 0): +==========test:9:23 start output========== +.TRY Exec r23 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:23 end output========== +.BG test line 9/job 24 finished (exit code: 0): +==========test:9:24 start output========== +.TRY Exec r24 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:24 end output========== +.BG test line 9/job 25 finished (exit code: 0): +==========test:9:25 start output========== +.TRY Exec r25 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:25 end output========== +.BG test line 9/job 26 finished (exit code: 0): +==========test:9:26 start output========== +.TRY Exec r26 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:26 end output========== +.BG test line 9/job 27 finished (exit code: 0): +==========test:9:27 start output========== +.TRY Exec r27 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:27 end output========== +.BG test line 9/job 28 finished (exit code: 0): +==========test:9:28 start output========== +.TRY Exec r28 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:28 end output========== +.BG test line 9/job 29 finished (exit code: 0): +==========test:9:29 start output========== +.TRY Exec r29 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:29 end output========== +.BG test line 9/job 30 finished (exit code: 0): +==========test:9:30 start output========== +.TRY Exec r30 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:30 end output========== +.BG test line 9/job 31 finished (exit code: 0): +==========test:9:31 start output========== +.TRY Exec r31 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:31 end output========== +.BG test line 9/job 32 finished (exit code: 0): +==========test:9:32 start output========== +.TRY Exec r32 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:32 end output========== +.BG test line 9/job 33 finished (exit code: 0): +==========test:9:33 start output========== +.TRY Exec r33 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:33 end output========== +.BG test line 9/job 34 finished (exit code: 0): +==========test:9:34 start output========== +.TRY Exec r34 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:34 end output========== +.BG test line 9/job 35 finished (exit code: 0): +==========test:9:35 start output========== +.TRY Exec r35 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:35 end output========== +.BG test line 9/job 36 finished (exit code: 0): +==========test:9:36 start output========== +.TRY Exec r36 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:36 end output========== +.BG test line 9/job 37 finished (exit code: 0): +==========test:9:37 start output========== +.TRY Exec r37 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:37 end output========== +.BG test line 9/job 38 finished (exit code: 0): +==========test:9:38 start output========== +.TRY Exec r38 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:38 end output========== +.BG test line 9/job 39 finished (exit code: 0): +==========test:9:39 start output========== +.TRY Exec r39 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:39 end output========== +.BG test line 9/job 40 finished (exit code: 0): +==========test:9:40 start output========== +.TRY Exec r40 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:40 end output========== +.BG test line 9/job 41 finished (exit code: 0): +==========test:9:41 start output========== +.TRY Exec r41 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:41 end output========== +.BG test line 9/job 42 finished (exit code: 0): +==========test:9:42 start output========== +.TRY Exec r42 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:42 end output========== +.BG test line 9/job 43 finished (exit code: 0): +==========test:9:43 start output========== +.TRY Exec r43 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:43 end output========== +.BG test line 9/job 44 finished (exit code: 0): +==========test:9:44 start output========== +.TRY Exec r44 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:44 end output========== +.BG test line 9/job 45 finished (exit code: 0): +==========test:9:45 start output========== +.TRY Exec r45 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:45 end output========== +.BG test line 9/job 46 finished (exit code: 0): +==========test:9:46 start output========== +.TRY Exec r46 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:46 end output========== +.BG test line 9/job 47 finished (exit code: 0): +==========test:9:47 start output========== +.TRY Exec r47 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:47 end output========== +.BG test line 9/job 48 finished (exit code: 0): +==========test:9:48 start output========== +.TRY Exec r48 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:48 end output========== +.BG test line 9/job 49 finished (exit code: 0): +==========test:9:49 start output========== +.TRY Exec r49 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:49 end output========== +.BG test line 9/job 50 finished (exit code: 0): +==========test:9:50 start output========== +.TRY Exec r50 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:50 end output========== +.BG test line 9/job 51 finished (exit code: 0): +==========test:9:51 start output========== +.TRY Exec r51 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:51 end output========== +.BG test line 9/job 52 finished (exit code: 0): +==========test:9:52 start output========== +.TRY Exec r52 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:52 end output========== +.BG test line 9/job 53 finished (exit code: 0): +==========test:9:53 start output========== +.TRY Exec r53 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:53 end output========== +.BG test line 9/job 54 finished (exit code: 0): +==========test:9:54 start output========== +.TRY Exec r54 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:54 end output========== +.BG test line 9/job 55 finished (exit code: 0): +==========test:9:55 start output========== +.TRY Exec r55 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:55 end output========== +.BG test line 9/job 56 finished (exit code: 0): +==========test:9:56 start output========== +.TRY Exec r56 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:56 end output========== +.BG test line 9/job 57 finished (exit code: 0): +==========test:9:57 start output========== +.TRY Exec r57 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:57 end output========== +.BG test line 9/job 58 finished (exit code: 0): +==========test:9:58 start output========== +.TRY Exec r58 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:58 end output========== +.BG test line 9/job 59 finished (exit code: 0): +==========test:9:59 start output========== +.TRY Exec r59 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:59 end output========== +.BG test line 9/job 60 finished (exit code: 0): +==========test:9:60 start output========== +.TRY Exec r60 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:60 end output========== +.BG test line 9/job 61 finished (exit code: 0): +==========test:9:61 start output========== +.TRY Exec r61 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:61 end output========== +.BG test line 9/job 62 finished (exit code: 0): +==========test:9:62 start output========== +.TRY Exec r62 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:62 end output========== +.BG test line 9/job 63 finished (exit code: 0): +==========test:9:63 start output========== +.TRY Exec r63 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:63 end output========== +.BG test line 9/job 64 finished (exit code: 0): +==========test:9:64 start output========== +.TRY Exec r64 op=start timeout=30000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:9:64 end output========== +.BG test line 13/job 1 finished (exit code: 0): +==========test:13:1 start output========== +.TRY Exec r1 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:1 end output========== +.BG test line 13/job 2 finished (exit code: 0): +==========test:13:2 start output========== +.TRY Exec r2 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:2 end output========== +.BG test line 13/job 3 finished (exit code: 0): +==========test:13:3 start output========== +.TRY Exec r3 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:3 end output========== +.BG test line 13/job 4 finished (exit code: 0): +==========test:13:4 start output========== +.TRY Exec r4 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:4 end output========== +.BG test line 13/job 5 finished (exit code: 0): +==========test:13:5 start output========== +.TRY Exec r5 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:5 end output========== +.BG test line 13/job 6 finished (exit code: 0): +==========test:13:6 start output========== +.TRY Exec r6 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:6 end output========== +.BG test line 13/job 7 finished (exit code: 0): +==========test:13:7 start output========== +.TRY Exec r7 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:7 end output========== +.BG test line 13/job 8 finished (exit code: 0): +==========test:13:8 start output========== +.TRY Exec r8 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:8 end output========== +.BG test line 13/job 9 finished (exit code: 0): +==========test:13:9 start output========== +.TRY Exec r9 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:9 end output========== +.BG test line 13/job 10 finished (exit code: 0): +==========test:13:10 start output========== +.TRY Exec r10 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:10 end output========== +.BG test line 13/job 11 finished (exit code: 0): +==========test:13:11 start output========== +.TRY Exec r11 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:11 end output========== +.BG test line 13/job 12 finished (exit code: 0): +==========test:13:12 start output========== +.TRY Exec r12 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:12 end output========== +.BG test line 13/job 13 finished (exit code: 0): +==========test:13:13 start output========== +.TRY Exec r13 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:13 end output========== +.BG test line 13/job 14 finished (exit code: 0): +==========test:13:14 start output========== +.TRY Exec r14 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:14 end output========== +.BG test line 13/job 15 finished (exit code: 0): +==========test:13:15 start output========== +.TRY Exec r15 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:15 end output========== +.BG test line 13/job 16 finished (exit code: 0): +==========test:13:16 start output========== +.TRY Exec r16 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:16 end output========== +.BG test line 13/job 17 finished (exit code: 0): +==========test:13:17 start output========== +.TRY Exec r17 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:17 end output========== +.BG test line 13/job 18 finished (exit code: 0): +==========test:13:18 start output========== +.TRY Exec r18 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:18 end output========== +.BG test line 13/job 19 finished (exit code: 0): +==========test:13:19 start output========== +.TRY Exec r19 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:19 end output========== +.BG test line 13/job 20 finished (exit code: 0): +==========test:13:20 start output========== +.TRY Exec r20 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:20 end output========== +.BG test line 13/job 21 finished (exit code: 0): +==========test:13:21 start output========== +.TRY Exec r21 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:21 end output========== +.BG test line 13/job 22 finished (exit code: 0): +==========test:13:22 start output========== +.TRY Exec r22 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:22 end output========== +.BG test line 13/job 23 finished (exit code: 0): +==========test:13:23 start output========== +.TRY Exec r23 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:23 end output========== +.BG test line 13/job 24 finished (exit code: 0): +==========test:13:24 start output========== +.TRY Exec r24 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:24 end output========== +.BG test line 13/job 25 finished (exit code: 0): +==========test:13:25 start output========== +.TRY Exec r25 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:25 end output========== +.BG test line 13/job 26 finished (exit code: 0): +==========test:13:26 start output========== +.TRY Exec r26 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:26 end output========== +.BG test line 13/job 27 finished (exit code: 0): +==========test:13:27 start output========== +.TRY Exec r27 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:27 end output========== +.BG test line 13/job 28 finished (exit code: 0): +==========test:13:28 start output========== +.TRY Exec r28 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:28 end output========== +.BG test line 13/job 29 finished (exit code: 0): +==========test:13:29 start output========== +.TRY Exec r29 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:29 end output========== +.BG test line 13/job 30 finished (exit code: 0): +==========test:13:30 start output========== +.TRY Exec r30 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:30 end output========== +.BG test line 13/job 31 finished (exit code: 0): +==========test:13:31 start output========== +.TRY Exec r31 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:31 end output========== +.BG test line 13/job 32 finished (exit code: 0): +==========test:13:32 start output========== +.TRY Exec r32 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:32 end output========== +.BG test line 13/job 33 finished (exit code: 0): +==========test:13:33 start output========== +.TRY Exec r33 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:33 end output========== +.BG test line 13/job 34 finished (exit code: 0): +==========test:13:34 start output========== +.TRY Exec r34 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:34 end output========== +.BG test line 13/job 35 finished (exit code: 0): +==========test:13:35 start output========== +.TRY Exec r35 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:35 end output========== +.BG test line 13/job 36 finished (exit code: 0): +==========test:13:36 start output========== +.TRY Exec r36 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:36 end output========== +.BG test line 13/job 37 finished (exit code: 0): +==========test:13:37 start output========== +.TRY Exec r37 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:37 end output========== +.BG test line 13/job 38 finished (exit code: 0): +==========test:13:38 start output========== +.TRY Exec r38 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:38 end output========== +.BG test line 13/job 39 finished (exit code: 0): +==========test:13:39 start output========== +.TRY Exec r39 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:39 end output========== +.BG test line 13/job 40 finished (exit code: 0): +==========test:13:40 start output========== +.TRY Exec r40 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:40 end output========== +.BG test line 13/job 41 finished (exit code: 0): +==========test:13:41 start output========== +.TRY Exec r41 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:41 end output========== +.BG test line 13/job 42 finished (exit code: 0): +==========test:13:42 start output========== +.TRY Exec r42 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:42 end output========== +.BG test line 13/job 43 finished (exit code: 0): +==========test:13:43 start output========== +.TRY Exec r43 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:43 end output========== +.BG test line 13/job 44 finished (exit code: 0): +==========test:13:44 start output========== +.TRY Exec r44 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:44 end output========== +.BG test line 13/job 45 finished (exit code: 0): +==========test:13:45 start output========== +.TRY Exec r45 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:45 end output========== +.BG test line 13/job 46 finished (exit code: 0): +==========test:13:46 start output========== +.TRY Exec r46 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:46 end output========== +.BG test line 13/job 47 finished (exit code: 0): +==========test:13:47 start output========== +.TRY Exec r47 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:47 end output========== +.BG test line 13/job 48 finished (exit code: 0): +==========test:13:48 start output========== +.TRY Exec r48 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:48 end output========== +.BG test line 13/job 49 finished (exit code: 0): +==========test:13:49 start output========== +.TRY Exec r49 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:49 end output========== +.BG test line 13/job 50 finished (exit code: 0): +==========test:13:50 start output========== +.TRY Exec r50 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:50 end output========== +.BG test line 13/job 51 finished (exit code: 0): +==========test:13:51 start output========== +.TRY Exec r51 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:51 end output========== +.BG test line 13/job 52 finished (exit code: 0): +==========test:13:52 start output========== +.TRY Exec r52 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:52 end output========== +.BG test line 13/job 53 finished (exit code: 0): +==========test:13:53 start output========== +.TRY Exec r53 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:53 end output========== +.BG test line 13/job 54 finished (exit code: 0): +==========test:13:54 start output========== +.TRY Exec r54 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:54 end output========== +.BG test line 13/job 55 finished (exit code: 0): +==========test:13:55 start output========== +.TRY Exec r55 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:55 end output========== +.BG test line 13/job 56 finished (exit code: 0): +==========test:13:56 start output========== +.TRY Exec r56 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:56 end output========== +.BG test line 13/job 57 finished (exit code: 0): +==========test:13:57 start output========== +.TRY Exec r57 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:57 end output========== +.BG test line 13/job 58 finished (exit code: 0): +==========test:13:58 start output========== +.TRY Exec r58 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:58 end output========== +.BG test line 13/job 59 finished (exit code: 0): +==========test:13:59 start output========== +.TRY Exec r59 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:59 end output========== +.BG test line 13/job 60 finished (exit code: 0): +==========test:13:60 start output========== +.TRY Exec r60 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:60 end output========== +.BG test line 13/job 61 finished (exit code: 0): +==========test:13:61 start output========== +.TRY Exec r61 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:61 end output========== +.BG test line 13/job 62 finished (exit code: 0): +==========test:13:62 start output========== +.TRY Exec r62 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:62 end output========== +.BG test line 13/job 63 finished (exit code: 0): +==========test:13:63 start output========== +.TRY Exec r63 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:63 end output========== +.BG test line 13/job 64 finished (exit code: 0): +==========test:13:64 start output========== +.TRY Exec r64 op=monitor timeout=30000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:13:64 end output========== diff --git a/lrm/test/testcases/metadata b/lrm/test/testcases/metadata new file mode 100644 index 0000000..d155757 --- /dev/null +++ b/lrm/test/testcases/metadata @@ -0,0 +1,29 @@ +# list various meta-data +%setenv LANG=POSIX +%extcheck sort +classes +%extcheck ra-list.sh +types class=ocf +%extcheck ra-list.sh +types class=lsb +%extcheck ra-list.sh +types class=heartbeat +#%extcheck ra-list.sh +#types class=stonith +%extcheck xmllint.sh many +classmeta class=ocf +%extcheck xmllint.sh many +classmeta class=lsb +%extcheck xmllint.sh many +classmeta class=heartbeat +#%extcheck xmllint.sh many +#classmeta class=stonith +%extcheck xmllint.sh +meta class=ocf type=Dummy +%extcheck xmllint.sh +meta class=lsb type=lrmregtest +%extcheck xmllint.sh +meta class=heartbeat type=Dummy +#%extcheck xmllint.sh +#meta class=stonith type=ssh +provider class=ocf type=IPaddr diff --git a/lrm/test/testcases/metadata.exp b/lrm/test/testcases/metadata.exp new file mode 100644 index 0000000..158bad2 --- /dev/null +++ b/lrm/test/testcases/metadata.exp @@ -0,0 +1,31 @@ +.SETENV LANG=POSIX +.EXTCHECK sort +.TRY List classes +There are 4 RA classes supported: +heartbeat +lsb +ocf +stonith +.EXTCHECK ra-list.sh +.TRY List types class=ocf +Cool. RA list passed. +.EXTCHECK ra-list.sh +.TRY List types class=lsb +Cool. RA list passed. +.EXTCHECK ra-list.sh +.TRY List types class=heartbeat +Cool. RA list passed. +.EXTCHECK xmllint.sh many +.TRY Meta-data class=ocf +.EXTCHECK xmllint.sh many +.TRY Meta-data class=lsb +.EXTCHECK xmllint.sh many +.TRY Meta-data class=heartbeat +.EXTCHECK xmllint.sh +.TRY Show meta-data class=ocf type=Dummy provider=heartbeat +.EXTCHECK xmllint.sh +.TRY Show meta-data class=lsb type=lrmregtest provider=heartbeat +.EXTCHECK xmllint.sh +.TRY Show meta-data class=heartbeat type=Dummy provider=heartbeat +.TRY Show provider class=ocf type=IPaddr +heartbeat diff --git a/lrm/test/testcases/ra-list.sh b/lrm/test/testcases/ra-list.sh new file mode 100755 index 0000000..38fb67b --- /dev/null +++ b/lrm/test/testcases/ra-list.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +awk ' +NR==1 {num=$3;next} +{in_num++} +END{ + if( num!=in_num ) + print "ERROR: A mismatch in number of reported RAs!"; + else + print "Cool. RA list passed."; +} +' diff --git a/lrm/test/testcases/rscexec b/lrm/test/testcases/rscexec new file mode 100644 index 0000000..e118ae1 --- /dev/null +++ b/lrm/test/testcases/rscexec @@ -0,0 +1,48 @@ +list +# ocf +%setenv dflt_rsc=rscexec_rsc_r1 +add rsc=rscexec_rsc_r1 args="delay=0" +list +exec operation=start +state +exec operation=monitor +exec operation=start +exec operation=monitor +exec operation=stop +state +exec operation=monitor +exec operation=stop +exec operation=monitor +exec operation=meta-data +del +# lsb +%setenv dflt_class=lsb dftl_rsc=rscexec_rsc_r1-lsb +add +exec operation=start +state +exec operation=monitor +exec operation=start +exec operation=monitor +exec operation=stop +state +exec operation=monitor +exec operation=stop +exec operation=monitor +exec operation=meta-data +del +%stop +# stonith +%setenv dflt_class=stonith dftl_rsc=rscexec_rsc_r1-stonith +add type=null args="hostlist=node1" +exec operation=start +state +exec operation=monitor +exec operation=start +exec operation=monitor +exec operation=stop +state +exec operation=monitor +exec operation=stop +exec operation=monitor +exec operation=meta-data +del diff --git a/lrm/test/testcases/rscexec.exp b/lrm/test/testcases/rscexec.exp new file mode 100644 index 0000000..71bdc2e --- /dev/null +++ b/lrm/test/testcases/rscexec.exp @@ -0,0 +1,117 @@ +.TRY List resources +Currently no resources are managed by LRM. +.SETENV dflt_rsc=rscexec_rsc_r1 +.TRY Add resource rscexec_rsc_r1 class=ocf type=lrmregtest provider=heartbeat args=delay=0 +Succeeded in adding this resource. +.TRY List resources + +Resource ID:rscexec_rsc_r1 +Resource agent class:ocf +Resource agent type:lrmregtest +Resource agent provider:heartbeat +Resource agent parameters:delay=0 +.TRY Exec rscexec_rsc_r1 op=start timeout=1000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +.TRY Show state rscexec_rsc_r1 +resource state:LRM_RSC_IDLE +The resource 1 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=0 +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=start timeout=1000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=stop timeout=1000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Show state rscexec_rsc_r1 +resource state:LRM_RSC_IDLE +The resource 3 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=0 + operation 'monitor' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=0 + operation 'stop' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=0 +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=7): [null] + +.TRY Exec rscexec_rsc_r1 op=stop timeout=1000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=7): [null] + +.TRY Exec rscexec_rsc_r1 op=meta-data timeout=1000 interval=0 target=EVERYTIME args= +> meta-data succeed (status=0,rc=0): [null] + +.TRY Delete resource rscexec_rsc_r1 +Succeeded in deleting this resource. +.SETENV dflt_class=lsb dftl_rsc=rscexec_rsc_r1-lsb +.TRY Add resource rscexec_rsc_r1 class=lsb type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY Exec rscexec_rsc_r1 op=start timeout=1000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +.TRY Show state rscexec_rsc_r1 +resource state:LRM_RSC_IDLE +The resource 1 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=start timeout=1000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=stop timeout=1000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Show state rscexec_rsc_r1 +resource state:LRM_RSC_IDLE +The resource 3 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: + operation 'monitor' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: + operation 'stop' [call_id=(removed)]: + start_delay=0, interval=0, timeout=1000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=7): [null] + +.TRY Exec rscexec_rsc_r1 op=stop timeout=1000 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Exec rscexec_rsc_r1 op=monitor timeout=1000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=7): [null] + +.TRY Exec rscexec_rsc_r1 op=meta-data timeout=1000 interval=0 target=EVERYTIME args= +> meta-data succeed (status=0,rc=0): [null] + +.TRY Delete resource rscexec_rsc_r1 +Succeeded in deleting this resource. diff --git a/lrm/test/testcases/rscmgmt b/lrm/test/testcases/rscmgmt new file mode 100644 index 0000000..8d745d3 --- /dev/null +++ b/lrm/test/testcases/rscmgmt @@ -0,0 +1,29 @@ +list +# add/remove resources +# +add rsc=r1 +info rsc=r1 +list +del rsc=r1 +%setenv dflt_class=lsb dflt_type=lrmregtest +list +add rsc=r1 +list +del rsc=r1 +list +# +# a bit of mix +# +%setenv dflt_class=ocf +add rsc=r1 +add rsc=r1 class=lsb type=lrmregtest +add rsc=r1 +del rsc=r1 +add rsc=r1 class=lsb type=lrmregtest +list +add rsc=r2 +list +del rsc=r1 +del rsc=r2 +list +del rsc=r1 diff --git a/lrm/test/testcases/rscmgmt.exp b/lrm/test/testcases/rscmgmt.exp new file mode 100644 index 0000000..3a5c4bf --- /dev/null +++ b/lrm/test/testcases/rscmgmt.exp @@ -0,0 +1,74 @@ +.TRY List resources +Currently no resources are managed by LRM. +.TRY Add resource r1 class=ocf type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY Show info r1 + +Resource ID:r1 +Resource agent class:ocf +Resource agent type:lrmregtest +Resource agent provider:heartbeat +.TRY List resources + +Resource ID:r1 +Resource agent class:ocf +Resource agent type:lrmregtest +Resource agent provider:heartbeat +.TRY Delete resource r1 +Succeeded in deleting this resource. +.SETENV dflt_class=lsb dflt_type=lrmregtest +.TRY List resources +Currently no resources are managed by LRM. +.TRY Add resource r1 class=lsb type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY List resources + +Resource ID:r1 +Resource agent class:lsb +Resource agent type:lrmregtest +Resource agent provider:heartbeat +.TRY Delete resource r1 +Succeeded in deleting this resource. +.TRY List resources +Currently no resources are managed by LRM. +.SETENV dflt_class=ocf +.TRY Add resource r1 class=ocf type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY Add resource r1 class=lsb type=lrmregtest provider=heartbeat args= +ERROR: lrm_add_rsc(): got a return code HA_FAIL from a reply message of addrsc with function get_ret_from_msg. +Failed to add this resource. +.TRY Add resource r1 class=ocf type=lrmregtest provider=heartbeat args= +ERROR: lrm_add_rsc(): got a return code HA_FAIL from a reply message of addrsc with function get_ret_from_msg. +Failed to add this resource. +.TRY Delete resource r1 +Succeeded in deleting this resource. +.TRY Add resource r1 class=lsb type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY List resources + +Resource ID:r1 +Resource agent class:lsb +Resource agent type:lrmregtest +Resource agent provider:heartbeat +.TRY Add resource r2 class=ocf type=lrmregtest provider=heartbeat args= +Succeeded in adding this resource. +.TRY List resources + +Resource ID:r2 +Resource agent class:ocf +Resource agent type:lrmregtest +Resource agent provider:heartbeat + +Resource ID:r1 +Resource agent class:lsb +Resource agent type:lrmregtest +Resource agent provider:heartbeat +.TRY Delete resource r1 +Succeeded in deleting this resource. +.TRY Delete resource r2 +Succeeded in deleting this resource. +.TRY List resources +Currently no resources are managed by LRM. +.TRY Delete resource r1 +ERROR: lrm_delete_rsc(): got a return code HA_FAIL from a reply message of delrsc with function get_ret_from_msg. +Failed to delete this resource. diff --git a/lrm/test/testcases/rscmgmt.log_filter b/lrm/test/testcases/rscmgmt.log_filter new file mode 100755 index 0000000..34debc5 --- /dev/null +++ b/lrm/test/testcases/rscmgmt.log_filter @@ -0,0 +1,13 @@ +#!/bin/sh + +awk ' +n<2 && /ERROR: on_msg_add_rsc: same id resource exists./ {n++; next} +m<1 && /ERROR: on_msg_del_rsc: no rsc with id/ {m++; next} +{print} +END{ + if( n!=2 ) + print "ERROR: missed on_msg_add_rsc errors"; + if( m!=1 ) + print "ERROR: missed on_msg_del_rsc errors"; +} +' diff --git a/lrm/test/testcases/serialize b/lrm/test/testcases/serialize new file mode 100644 index 0000000..cad96b3 --- /dev/null +++ b/lrm/test/testcases/serialize @@ -0,0 +1,33 @@ +list +# allow for a delay of 2 seconds +%setenv dflt_timeout=2500 +add rsc=r1 args="delay=2" +# +# we run the next three ops in the background +# in case ops are not serialized, the lrmregtest RA should complain +# +%bg 2 +exec operation=start +# insert sleeps to make sure that the operations are started in +# the order given here +%sleep 1 +# set timeouts high enough so that no op fails +exec operation=start timeout=3000 +%sleep 1 +%bgrepeat 4 +exec operation=monitor timeout=11000 +%sleep 11 +state +exec operation=stop +state +del rsc=r1 +# +# +# +%setenv dflt_rsc=r2 dflt_timeout=10500 +add rsc=r2 args="ignore_TERM=1 delay=9" +exec operation=start +%bg +exec operation=monitor timeout=500 +exec operation=monitor +del rsc=r2 diff --git a/lrm/test/testcases/serialize.exp b/lrm/test/testcases/serialize.exp new file mode 100644 index 0000000..b290c95 --- /dev/null +++ b/lrm/test/testcases/serialize.exp @@ -0,0 +1,100 @@ +.TRY List resources +Currently no resources are managed by LRM. +.SETENV dflt_timeout=2500 +.TRY Add resource r1 class=ocf type=lrmregtest provider=heartbeat args=delay=2 +Succeeded in adding this resource. +.BG 2 +.BG test line 10/job 1 runs in background +.SLEEP 1 +.BG test line 15/job 2 runs in background +.SLEEP 1 +.BGREPEAT 4 +.BG test line 18/job 1 runs in background +.BG test line 18/job 2 runs in background +.BG test line 18/job 3 runs in background +.BG test line 18/job 4 runs in background +.SLEEP 11 +.TRY Show state r1 +resource state:LRM_RSC_IDLE +The resource 2 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=3000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=2 + operation 'monitor' [call_id=(removed)]: + start_delay=0, interval=0, timeout=11000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=2 +.TRY Exec r1 op=stop timeout=2500 interval=0 target=EVERYTIME args= +> stop succeed (status=0,rc=0): [null] + +.TRY Show state r1 +resource state:LRM_RSC_IDLE +The resource 3 operations' information: + operation 'start' [call_id=(removed)]: + start_delay=0, interval=0, timeout=3000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=2 + operation 'monitor' [call_id=(removed)]: + start_delay=0, interval=0, timeout=11000, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=2 + operation 'stop' [call_id=(removed)]: + start_delay=0, interval=0, timeout=2500, app_name=lrmadmin + rc=0 (ok), op_status=0 (succeed) + parameters: delay=2 +.TRY Delete resource r1 +Succeeded in deleting this resource. +.SETENV dflt_rsc=r2 dflt_timeout=10500 +.TRY Add resource r2 class=ocf type=lrmregtest provider=heartbeat args=ignore_TERM=1 delay=9 +Succeeded in adding this resource. +.TRY Exec r2 op=start timeout=10500 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +.BG +.BG test line 31/job 1 runs in background +.TRY Exec r2 op=monitor timeout=10500 interval=0 target=EVERYTIME args= +ERROR: This operation has timed out - no result from lrmd. +.TRY Delete resource r2 +Succeeded in deleting this resource. +.BG test line 10/job 1 finished (exit code: 0): +==========test:10:1 start output========== +.TRY Exec r1 op=start timeout=2500 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:10:1 end output========== +.BG test line 15/job 2 finished (exit code: 0): +==========test:15:2 start output========== +.TRY Exec r1 op=start timeout=3000 interval=0 target=EVERYTIME args= +> start succeed (status=0,rc=0): [null] + +==========test:15:2 end output========== +.BG test line 18/job 1 finished (exit code: 0): +==========test:18:1 start output========== +.TRY Exec r1 op=monitor timeout=11000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:18:1 end output========== +.BG test line 18/job 2 finished (exit code: 0): +==========test:18:2 start output========== +.TRY Exec r1 op=monitor timeout=11000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:18:2 end output========== +.BG test line 18/job 3 finished (exit code: 0): +==========test:18:3 start output========== +.TRY Exec r1 op=monitor timeout=11000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:18:3 end output========== +.BG test line 18/job 4 finished (exit code: 0): +==========test:18:4 start output========== +.TRY Exec r1 op=monitor timeout=11000 interval=0 target=EVERYTIME args= +> monitor succeed (status=0,rc=0): [null] + +==========test:18:4 end output========== +.BG test line 31/job 1 finished (exit code: 0): +==========test:31:1 start output========== +.TRY Exec r2 op=monitor timeout=500 interval=0 target=EVERYTIME args= +ERROR: This operation has timed out - no result from lrmd. +==========test:31:1 end output========== diff --git a/lrm/test/testcases/stonith b/lrm/test/testcases/stonith new file mode 100644 index 0000000..f21cf18 --- /dev/null +++ b/lrm/test/testcases/stonith @@ -0,0 +1,2 @@ +%extcheck xmllint.sh many +%shell stonith -L | while read p; do echo $p:heartbeat; stonith -m -t $p; done diff --git a/lrm/test/testcases/stonith.exp b/lrm/test/testcases/stonith.exp new file mode 100644 index 0000000..f9f1042 --- /dev/null +++ b/lrm/test/testcases/stonith.exp @@ -0,0 +1,2 @@ +.EXTCHECK xmllint.sh many +.SHELL stonith -L | while read p; do echo $p:heartbeat; stonith -m -t $p; done diff --git a/lrm/test/testcases/xmllint.sh b/lrm/test/testcases/xmllint.sh new file mode 100755 index 0000000..f61288c --- /dev/null +++ b/lrm/test/testcases/xmllint.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +gawk -v many="$1" ' +BEGIN{XMLLINT="xmllint --noout -";} +function chkoutput(ra) { + if( ra=="" ) return; + if( close(XMLLINT) ) # we need gawk for this + print "xmllint reported error in RA:",ra; +} +many=="many" && /^[a-zA-Z][^:]*:[a-zA-Z0-9]+$/ { + chkoutput(ra); + ra=$0; + next; +} +{ print | XMLLINT } +END{ + if( many!="many" ) + chkoutput("noname"); +} +' |