#!@PYTHON@ -tt import sys import time import atexit import logging import inspect import requests.exceptions sys.path.append("@FENCEAGENTSLIBDIR@") from fencing import * from fencing import fail_usage, is_executable, run_command, run_delay override_status = "" EVACUABLE_TAG = "evacuable" TRUE_TAGS = ['true'] def get_power_status(connection, options): if len(override_status): logging.debug("Pretending we're " + override_status) return override_status status = "unknown" logging.debug("get action: " + options["--action"]) if connection: try: services = connection.services.list(host=options["--plug"], binary="nova-compute") for service in services: logging.debug("Status of %s on %s is %s, %s" % (service.binary, options["--plug"], service.state, service.status)) if service.state == "up" and service.status == "enabled": # Up and operational status = "on" elif service.state == "down" and service.status == "disabled": # Down and fenced status = "off" elif service.state == "down": # Down and requires fencing status = "failed" elif service.state == "up": # Up and requires unfencing status = "running" else: logging.warning("Unknown status detected from nova for %s: %s, %s" % (options["--plug"], service.state, service.status)) status = "%s %s" % (service.state, service.status) break except requests.exception.ConnectionError as err: logging.warning("Nova connection failed: " + str(err)) logging.debug("Final status of %s is %s" % (options["--plug"], status)) return status def get_power_status_simple(connection, options): status = get_power_status(connection, options) if status in [ "off" ]: return status return "on" def set_attrd_status(host, status, options): logging.debug("Setting fencing status for %s to %s" % (host, status)) run_command(options, "attrd_updater -p -n evacuate -Q -N %s -U %s" % (host, status)) def get_attrd_status(host, options): (status, pipe_stdout, pipe_stderr) = run_command(options, "attrd_updater -p -n evacuate -Q -N %s" % (host)) fields = pipe_stdout.split('"') if len(fields) > 6: return fields[5] logging.debug("Got %s: o:%s e:%s n:%d" % (status, pipe_stdout, pipe_stderr, len(fields))) return "" def set_power_status_on(connection, options): # Wait for any evacuations to complete while True: current = get_attrd_status(options["--plug"], options) if current in ["no", ""]: logging.info("Evacuation complete for: %s '%s'" % (options["--plug"], current)) break else: logging.info("Waiting for %s to complete evacuations: %s" % (options["--plug"], current)) time.sleep(2) status = get_power_status(connection, options) # Should we do it for 'failed' too? if status in [ "off", "running", "failed" ]: try: # Forcing the host back up logging.info("Forcing nova-compute back up on "+options["--plug"]) connection.services.force_down(options["--plug"], "nova-compute", force_down=False) logging.info("Forced nova-compute back up on "+options["--plug"]) except Exception as e: # In theory, if force_down=False fails, that's for the exact # same possible reasons that below with force_down=True # eg. either an incompatible version or an old client. # Since it's about forcing back to a default value, there is # no real worries to just consider it's still okay even if the # command failed logging.warn("Exception from attempt to force " "host back up via nova API: " "%s: %s" % (e.__class__.__name__, e)) # Forcing the service back up in case it was disabled logging.info("Enabling nova-compute on "+options["--plug"]) connection.services.enable(options["--plug"], 'nova-compute') # Pretend we're 'on' so that the fencing library doesn't loop forever waiting for the node to boot override_status = "on" elif status not in ["on"]: # Not safe to unfence, don't waste time looping to see if the status changes to "on" options["--power-timeout"] = "0" def set_power_status_off(connection, options): status = get_power_status(connection, options) if status in [ "off" ]: return try: # Until 2.53 connection.services.force_down( options["--plug"], "nova-compute", force_down=True) connection.services.disable(options["--plug"], 'nova-compute') except Exception as e: # Something went wrong when we tried to force the host down. # That could come from either an incompatible API version # eg. UnsupportedVersion or VersionNotFoundForAPIMethod # or because novaclient is old and doesn't include force_down yet # eg. AttributeError # In that case, fallbacking to wait for Nova to catch the right state. logging.error("Exception from attempt to force host down via nova API: " "%s: %s" % (e.__class__.__name__, e)) # need to wait for nova to update its internal status or we # cannot call host-evacuate while get_power_status(connection, options) not in ["off"]: # Loop forever if need be. # # Some callers (such as Pacemaker) will have a timer # running and kill us if necessary logging.debug("Waiting for nova to update its internal state for %s" % options["--plug"]) time.sleep(1) set_attrd_status(options["--plug"], "yes", options) def set_power_status(connection, options): global override_status override_status = "" logging.debug("set action: " + options["--action"]) if not connection: return if options["--action"] in ["off", "reboot"]: set_power_status_off(connection, options) else: set_power_status_on(connection, options) logging.debug("set action passed: " + options["--action"]) sys.exit(0) def fix_domain(connection, options): domains = {} last_domain = None if connection: # Find it in nova services = connection.services.list(binary="nova-compute") for service in services: shorthost = service.host.split('.')[0] if shorthost == service.host: # Nova is not using FQDN calculated = "" else: # Compute nodes are named as FQDN, strip off the hostname calculated = service.host.replace(shorthost+".", "") if calculated == last_domain: # Avoid complaining for each compute node with the same name # One hopes they don't appear interleaved as A.com B.com A.com B.com logging.debug("Calculated the same domain from: %s" % service.host) continue domains[calculated] = service.host last_domain = calculated if "--domain" in options and options["--domain"] != calculated: # Warn in case nova isn't available at some point logging.warning("Supplied domain '%s' does not match the one calculated from: %s" % (options["--domain"], service.host)) if len(domains) == 0 and "--domain" not in options: logging.error("Could not calculate the domain names used by compute nodes in nova") elif len(domains) == 1 and "--domain" not in options: options["--domain"] = last_domain elif len(domains) == 1 and options["--domain"] != last_domain: logging.error("Overriding supplied domain '%s' as it does not match the one calculated from: %s" % (options["--domain"], domains[last_domain])) options["--domain"] = last_domain elif len(domains) > 1: logging.error("The supplied domain '%s' did not match any used inside nova: %s" % (options["--domain"], repr(domains))) sys.exit(1) return last_domain def fix_plug_name(connection, options): if options["--action"] == "list": return if "--plug" not in options: return calculated = fix_domain(connection, options) if calculated is None or "--domain" not in options: # Nothing supplied and nova not available... what to do... nothing return short_plug = options["--plug"].split('.')[0] logging.debug("Checking target '%s' against calculated domain '%s'"% (options["--plug"], calculated)) if options["--domain"] == "": # Ensure any domain is stripped off since nova isn't using FQDN options["--plug"] = short_plug elif options["--plug"].endswith(options["--domain"]): # Plug already uses the domain, don't re-add return else: # Add the domain to the plug options["--plug"] = short_plug + "." + options["--domain"] def get_plugs_list(connection, options): result = {} if connection: services = connection.services.list(binary="nova-compute") for service in services: longhost = service.host shorthost = longhost.split('.')[0] result[longhost] = ("", None) result[shorthost] = ("", None) return result def create_nova_connection(options): nova = None try: from novaclient import client from novaclient.exceptions import NotAcceptable except ImportError: fail_usage("Nova not found or not accessible") from keystoneauth1 import loading from keystoneauth1 import session from keystoneclient import discover # Prefer the oldest and strip the leading 'v' keystone_versions = discover.available_versions(options["--auth-url"]) keystone_version = keystone_versions[0]['id'][1:] kwargs = dict( auth_url=options["--auth-url"], username=options["--username"], password=options["--password"] ) if discover.version_match("2", keystone_version): kwargs["tenant_name"] = options["--tenant-name"] elif discover.version_match("3", keystone_version): kwargs["project_name"] = options["--tenant-name"] kwargs["user_domain_name"] = options["--user-domain"] kwargs["project_domain_name"] = options["--project-domain"] loader = loading.get_plugin_loader('password') keystone_auth = loader.load_from_options(**kwargs) keystone_session = session.Session(auth=keystone_auth, verify=not "--insecure" in options) nova_versions = [ "2.11", "2" ] for version in nova_versions: clientargs = inspect.getargspec(client.Client).varargs # Some versions of Openstack prior to Ocata only # supported positional arguments for username, # password, and tenant. # # Versions since Ocata only support named arguments. # # So we need to use introspection to figure out how to # create a Nova client. # # Happy days # if clientargs: # OSP < 11 # ArgSpec(args=['version', 'username', 'password', 'project_id', 'auth_url'], # varargs=None, # keywords='kwargs', defaults=(None, None, None, None)) nova = client.Client(version, None, # User None, # Password None, # Tenant None, # Auth URL insecure="--insecure" in options, region_name=options["--region-name"], endpoint_type=options["--endpoint-type"], session=keystone_session, auth=keystone_auth, http_log_debug="--verbose" in options) else: # OSP >= 11 # ArgSpec(args=['version'], varargs='args', keywords='kwargs', defaults=None) nova = client.Client(version, region_name=options["--region-name"], endpoint_type=options["--endpoint-type"], session=keystone_session, auth=keystone_auth, http_log_debug="--verbose" in options) try: nova.hypervisors.list() return nova except NotAcceptable as e: logging.warning(e) except Exception as e: logging.warning("Nova connection failed. %s: %s" % (e.__class__.__name__, e)) logging.warning("Couldn't obtain a supported connection to nova, tried: %s\n" % repr(nova_versions)) return None def define_new_opts(): all_opt["endpoint_type"] = { "getopt" : "e:", "longopt" : "endpoint-type", "help" : "-e, --endpoint-type=[endpoint] Nova Endpoint type (publicURL, internalURL, adminURL)", "required" : "0", "shortdesc" : "Nova Endpoint type", "default" : "internalURL", "order": 1, } all_opt["tenant_name"] = { "getopt" : "t:", "longopt" : "tenant-name", "help" : "-t, --tenant-name=[name] Keystone v2 Tenant or v3 Project Name", "required" : "0", "shortdesc" : "Keystone Admin Tenant or v3 Project", "default" : "", "order": 1, } all_opt["user-domain"] = { "getopt" : "u:", "longopt" : "user-domain", "help" : "-u, --user-domain=[name] Keystone v3 User Domain", "required" : "0", "shortdesc" : "Keystone v3 User Domain", "default" : "Default", "order": 2, } all_opt["project-domain"] = { "getopt" : "P:", "longopt" : "project-domain", "help" : "-P, --project-domain=[name] Keystone v3 Project Domain", "required" : "0", "shortdesc" : "Keystone v3 Project Domain", "default" : "Default", "order": 2, } all_opt["auth_url"] = { "getopt" : "k:", "longopt" : "auth-url", "help" : "-k, --auth-url=[url] Keystone Admin Auth URL", "required" : "0", "shortdesc" : "Keystone Admin Auth URL", "default" : "", "order": 1, } all_opt["region_name"] = { "getopt" : ":", "longopt" : "region-name", "help" : "--region-name=[region] Region Name", "required" : "0", "shortdesc" : "Region Name", "default" : "", "order": 1, } all_opt["insecure"] = { "getopt" : "", "longopt" : "insecure", "help" : "--insecure Explicitly allow agent to perform \"insecure\" TLS (https) requests", "required" : "0", "shortdesc" : "Allow Insecure TLS Requests", "order": 2, } all_opt["domain"] = { "getopt" : "d:", "longopt" : "domain", "help" : "-d, --domain=[string] DNS domain in which hosts live, useful when the cluster uses short names and nova uses FQDN", "required" : "0", "shortdesc" : "DNS domain in which hosts live", "order": 5, } all_opt["record_only"] = { "getopt" : "r:", "longopt" : "record-only", "help" : "--record-only Record the target as needing evacuation but as yet do not intiate it", "required" : "0", "shortdesc" : "Only record the target as needing evacuation", "default" : "False", "order": 5, } all_opt["instance_filtering"] = { "getopt" : "", "longopt" : "instance-filtering", "help" : "--instance-filtering Allow instances created from images and flavors with evacuable=true to be evacuated (or all if no images/flavors have been tagged)", "required" : "0", "shortdesc" : "Allow instances to be evacuated", "default" : "True", "order": 5, } all_opt["no_shared_storage"] = { "getopt" : "", "longopt" : "no-shared-storage", "help" : "--no-shared-storage Disable functionality for shared storage", "required" : "0", "shortdesc" : "Disable functionality for dealing with shared storage", "default" : "False", "order": 5, } all_opt["compute-domain"] = { "getopt" : ":", "longopt" : "compute-domain", "help" : "--compute-domain=[string] Replaced by --domain", "required" : "0", "shortdesc" : "Replaced by domain", "order": 6, } def set_multi_power_fn(connection, options, set_power_fn, get_power_fn, retry_attempts=1): for _ in range(retry_attempts): set_power_fn(connection, options) time.sleep(int(options["--power-wait"])) for _ in range(int(options["--power-timeout"])): if get_power_fn(connection, options) != options["--action"]: time.sleep(1) else: return True return False def main(): global override_status atexit.register(atexit_handler) device_opt = ["login", "passwd", "tenant_name", "auth_url", "fabric_fencing", "no_login", "no_password", "port", "domain", "compute-domain", "project-domain", "user-domain", "no_shared_storage", "endpoint_type", "record_only", "instance_filtering", "insecure", "region_name"] define_new_opts() all_opt["shell_timeout"]["default"] = "180" options = check_input(device_opt, process_input(device_opt)) docs = {} docs["shortdesc"] = "Fence agent for the automatic resurrection of OpenStack compute instances" docs["longdesc"] = "Used to tell Nova that compute nodes are down and to reschedule flagged instances" docs["vendorurl"] = "" show_docs(options, docs) if options["--record-only"] in [ "2", "Disabled", "disabled" ]: sys.exit(0) run_delay(options) # workaround to avoid regressions if "--compute-domain" in options and options["--compute-domain"]: options["--domain"] = options["--compute-domain"] del options["--domain"] # Disable insecure-certificate-warning message if "--insecure" in options: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.debug("Running "+options["--action"]) connection = create_nova_connection(options) if options["--action"] in ["off", "on", "reboot", "status"]: fix_plug_name(connection, options) if options["--action"] in ["reboot"]: options["--action"]="off" if options["--action"] in ["off", "on"]: # No status first, call our own version result = not set_multi_power_fn(connection, options, set_power_status, get_power_status_simple, 1 + int(options["--retry-on"])) elif options["--action"] in ["monitor"]: result = 0 else: result = fence_action(connection, options, set_power_status, get_power_status_simple, get_plugs_list, None) logging.debug("Result for "+options["--action"]+": "+repr(result)) if result == None: result = 0 sys.exit(result) if __name__ == "__main__": main()