#!/usr/bin/python # # Copyright (c) 2019 Zim Kalinowski, (@zikalino) # # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function __metaclass__ = type DOCUMENTATION = ''' --- module: azure_rm_hdinsightcluster version_added: "0.1.2" short_description: Manage Azure HDInsight Cluster instance description: - Create, update and delete instance of Azure HDInsight Cluster. options: resource_group: description: - The name of the resource group. required: True type: str name: description: - The name of the cluster. required: True type: str location: description: - Resource location. If not set, location from the resource group will be used as default. type: str cluster_version: description: - The version of the cluster. For example C(3.6). type: str os_type: description: - The type of operating system. type: str choices: - 'linux' tier: description: - The cluster tier. type: str choices: - 'standard' - 'premium' cluster_definition: description: - The cluster definition. type: dict suboptions: kind: description: - The type of cluster. type: str choices: - hadoop - spark - hbase - storm gateway_rest_username: description: - Gateway REST user name. type: str gateway_rest_password: description: - Gateway REST password. type: str compute_profile_roles: description: - The list of roles in the cluster. type: list elements: dict suboptions: name: description: - The name of the role. type: str choices: - 'headnode' - 'workernode' - 'zookepernode' min_instance_count: description: - The minimum instance count of the cluster. type: int target_instance_count: description: - The instance count of the cluster. type: int vm_size: description: - The size of the VM. type: str linux_profile: description: - The Linux OS profile. type: dict suboptions: username: description: - SSH user name. type: str password: description: - SSH password. type: str storage_accounts: description: - The list of storage accounts in the cluster. type: list elements: dict suboptions: name: description: - Blob storage endpoint. For example storage_account_name.blob.core.windows.net. type: str is_default: description: - Whether or not the storage account is the default storage account. type: bool container: description: - The container in the storage account. type: str key: description: - The storage account access key. type: str state: description: - Assert the state of the cluster. - Use C(present) to create or update a cluster and C(absent) to delete it. default: present type: str choices: - absent - present extends_documentation_fragment: - azure.azcollection.azure - azure.azcollection.azure_tags author: - Zim Kalinowski (@zikalino) ''' EXAMPLES = ''' - name: Create instance of HDInsight Cluster azure_rm_hdinsightcluster: resource_group: myResourceGroup name: myCluster location: eastus2 cluster_version: 3.6 os_type: linux tier: standard cluster_definition: kind: spark gateway_rest_username: http-user gateway_rest_password: MuABCPassword!!@123 storage_accounts: - name: myStorageAccount.blob.core.windows.net is_default: true container: myContainer key: GExmaxH4lDNdHA9nwAsCt8t4AOQas2y9vXQP1kKALTram7Q3/5xLVIab3+nYG1x63Xyak9/VXxQyNBHA9pDWw== compute_profile_roles: - name: headnode target_instance_count: 2 vm_size: Standard_D3 linux_profile: username: sshuser password: MuABCPassword!!@123 - name: workernode target_instance_count: 2 vm_size: Standard_D3 linux_profile: username: sshuser password: MuABCPassword!!@123 ''' RETURN = ''' id: description: - Fully qualified resource id of the cluster. returned: always type: str sample: /subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroups/myResourceGroup/providers/Microsoft.HDInsight/clusters/myCluster ''' from ansible_collections.azure.azcollection.plugins.module_utils.azure_rm_common import AzureRMModuleBase try: from azure.core.polling import LROPoller from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.hdinsight import HDInsightManagementClient except ImportError: # This is handled in azure_rm_common pass class Actions: NoAction, Create, Update, Delete = range(4) class AzureRMClusters(AzureRMModuleBase): """Configuration class for an Azure RM Cluster resource""" def __init__(self): self.module_arg_spec = dict( resource_group=dict( type='str', required=True ), name=dict( type='str', required=True ), location=dict( type='str' ), cluster_version=dict( type='str' ), os_type=dict( type='str', choices=['linux'] ), tier=dict( type='str', choices=['standard', 'premium'] ), cluster_definition=dict( type='dict' ), compute_profile_roles=dict( type='list', elements='dict', options=dict( name=dict(type='str', choices=['headnode', 'workernode', 'zookepernode']), min_instance_count=dict(type='int'), target_instance_count=dict(type='int'), vm_size=dict(type='str'), linux_profile=dict( type='dict', options=dict( username=dict(type='str'), password=dict(type='str', no_log=True) ) ), ) ), storage_accounts=dict( type='list', elements='dict', options=dict( name=dict(type='str'), is_default=dict(type='bool'), container=dict(type='str'), key=dict(type='str', no_log=True) ) ), state=dict( type='str', default='present', choices=['present', 'absent'] ) ) self.resource_group = None self.name = None self.parameters = dict() self.results = dict(changed=False) self.mgmt_client = None self.state = None self.to_do = Actions.NoAction self.tags_changed = False self.new_instance_count = None super(AzureRMClusters, self).__init__(derived_arg_spec=self.module_arg_spec, supports_check_mode=True, supports_tags=True) def exec_module(self, **kwargs): """Main module execution method""" for key in list(self.module_arg_spec.keys()) + ['tags']: if hasattr(self, key): setattr(self, key, kwargs[key]) elif kwargs[key] is not None: self.parameters[key] = kwargs[key] dict_expand(self.parameters, ['cluster_version'], 'properties') dict_camelize(self.parameters, ['os_type'], True) dict_expand(self.parameters, ['os_type'], 'properties') dict_camelize(self.parameters, ['tier'], True) dict_expand(self.parameters, ['tier'], 'properties') dict_rename(self.parameters, ['cluster_definition', 'gateway_rest_username'], 'restAuthCredential.username') dict_rename(self.parameters, ['cluster_definition', 'gateway_rest_password'], 'restAuthCredential.password') dict_expand(self.parameters, ['cluster_definition', 'restAuthCredential.username'], 'gateway') dict_expand(self.parameters, ['cluster_definition', 'restAuthCredential.password'], 'gateway') dict_expand(self.parameters, ['cluster_definition', 'gateway'], 'configurations') dict_expand(self.parameters, ['cluster_definition'], 'properties') dict_expand(self.parameters, ['compute_profile_roles', 'vm_size'], 'hardware_profile') dict_rename(self.parameters, ['compute_profile_roles', 'linux_profile'], 'linux_operating_system_profile') dict_expand(self.parameters, ['compute_profile_roles', 'linux_operating_system_profile'], 'os_profile') dict_rename(self.parameters, ['compute_profile_roles'], 'roles') dict_expand(self.parameters, ['roles'], 'compute_profile') dict_expand(self.parameters, ['compute_profile'], 'properties') dict_rename(self.parameters, ['storage_accounts'], 'storageaccounts') dict_expand(self.parameters, ['storageaccounts'], 'storage_profile') dict_expand(self.parameters, ['storage_profile'], 'properties') response = None self.mgmt_client = self.get_mgmt_svc_client(HDInsightManagementClient, is_track2=True, base_url=self._cloud_environment.endpoints.resource_manager) resource_group = self.get_resource_group(self.resource_group) if "location" not in self.parameters: self.parameters["location"] = resource_group.location old_response = self.get_cluster() if not old_response: self.log("Cluster instance doesn't exist") if self.state == 'absent': self.log("Old instance didn't exist") else: self.to_do = Actions.Create else: self.log("Cluster instance already exists") if self.state == 'absent': self.to_do = Actions.Delete elif self.state == 'present': compare_result = {} if (not default_compare(self.parameters, old_response, '', compare_result)): if compare_result.pop('/properties/compute_profile/roles/*/target_instance_count', False): # check if it's workernode new_count = 0 old_count = 0 for role in self.parameters['properties']['compute_profile']['roles']: if role['name'] == 'workernode': new_count = role['target_instance_count'] for role in old_response['properties']['compute_profile']['roles']: if role['name'] == 'workernode': old_count = role['target_instance_count'] if old_count != new_count: self.new_instance_count = new_count self.to_do = Actions.Update if compare_result.pop('/tags', False): self.to_do = Actions.Update self.tags_changed = True if compare_result: for k in compare_result.keys(): self.module.warn("property '" + k + "' cannot be updated (" + compare_result[k] + ")") self.module.warn("only tags and target_instance_count can be updated") if (self.to_do == Actions.Create) or (self.to_do == Actions.Update): self.log("Need to Create / Update the Cluster instance") self.results['changed'] = True if self.check_mode: return self.results response = self.create_update_cluster() self.log("Creation / Update done") elif self.to_do == Actions.Delete: self.log("Cluster instance deleted") self.results['changed'] = True if self.check_mode: return self.results self.delete_cluster() else: self.log("Cluster instance unchanged") self.results['changed'] = False response = old_response if self.state == 'present': self.results.update(self.format_item(response)) return self.results def create_update_cluster(self): ''' Creates or updates Cluster with the specified configuration. :return: deserialized Cluster instance state dictionary ''' self.log("Creating / Updating the Cluster instance {0}".format(self.name)) try: if self.to_do == Actions.Create: response = self.mgmt_client.clusters.begin_create(resource_group_name=self.resource_group, cluster_name=self.name, parameters=self.parameters) if isinstance(response, LROPoller): response = self.get_poller_result(response) else: if self.tags_changed: response = self.mgmt_client.clusters.update(resource_group_name=self.resource_group, cluster_name=self.name, parameters={'tags': self.parameters.get('tags')}) if isinstance(response, LROPoller): response = self.get_poller_result(response) if self.new_instance_count: response = self.mgmt_client.clusters.begin_resize(resource_group_name=self.resource_group, cluster_name=self.name, role_name='workernode', parameters={'target_instance_count': self.new_instance_count}) if isinstance(response, LROPoller): response = self.get_poller_result(response) except Exception as exc: self.fail("Error creating or updating Cluster instance: {0}".format(str(exc))) return response.as_dict() if response else {} def delete_cluster(self): ''' Deletes specified Cluster instance in the specified subscription and resource group. :return: True ''' self.log("Deleting the Cluster instance {0}".format(self.name)) try: response = self.mgmt_client.clusters.begin_delete(resource_group_name=self.resource_group, cluster_name=self.name) except Exception as e: self.fail("Error deleting the Cluster instance: {0}".format(str(e))) return True def get_cluster(self): ''' Gets the properties of the specified Cluster. :return: deserialized Cluster instance state dictionary ''' self.log("Checking if the Cluster instance {0} is present".format(self.name)) found = False try: response = self.mgmt_client.clusters.get(resource_group_name=self.resource_group, cluster_name=self.name) found = True self.log("Response : {0}".format(response)) self.log("Cluster instance : {0} found".format(response.name)) except ResourceNotFoundError as e: self.log('Did not find the Cluster instance.') if found is True: return response.as_dict() return False def format_item(self, d): d = { 'id': d.get('id', None) } return d def default_compare(new, old, path, result): if new is None: match = True elif isinstance(new, dict): match = True if not isinstance(old, dict): result[path] = 'old dict is null' match = False else: for k in new.keys(): if not default_compare(new.get(k), old.get(k, None), path + '/' + k, result): match = False elif isinstance(new, list): if not isinstance(old, list) or len(new) != len(old): result[path] = 'length is different or null' match = False elif len(old) == 0: match = True else: match = True if isinstance(old[0], dict): key = None if 'id' in old[0] and 'id' in new[0]: key = 'id' elif 'name' in old[0] and 'name' in new[0]: key = 'name' else: key = list(old[0])[0] new = sorted(new, key=lambda x: x.get(key, '')) old = sorted(old, key=lambda x: x.get(key, '')) else: new = sorted(new) old = sorted(old) for i in range(len(new)): if not default_compare(new[i], old[i], path + '/*', result): match = False return match else: if path.endswith('password'): match = True else: if path == '/location' or path.endswith('location_name'): new = new.replace(' ', '').lower() old = new.replace(' ', '').lower() if new == old: match = True else: result[path] = str(new) + ' != ' + str(old) match = False return match def dict_camelize(d, path, camelize_first): if isinstance(d, list): for i in range(len(d)): dict_camelize(d[i], path, camelize_first) elif isinstance(d, dict): if len(path) == 1: old_value = d.get(path[0], None) if old_value is not None: d[path[0]] = _snake_to_camel(old_value, camelize_first) else: sd = d.get(path[0], None) if sd is not None: dict_camelize(sd, path[1:], camelize_first) def dict_upper(d, path): if isinstance(d, list): for i in range(len(d)): dict_upper(d[i], path) elif isinstance(d, dict): if len(path) == 1: old_value = d.get(path[0], None) if old_value is not None: d[path[0]] = old_value.upper() else: sd = d.get(path[0], None) if sd is not None: dict_upper(sd, path[1:]) def dict_rename(d, path, new_name): if isinstance(d, list): for i in range(len(d)): dict_rename(d[i], path, new_name) elif isinstance(d, dict): if len(path) == 1: old_value = d.pop(path[0], None) if old_value is not None: d[new_name] = old_value else: sd = d.get(path[0], None) if sd is not None: dict_rename(sd, path[1:], new_name) def dict_expand(d, path, outer_dict_name): if isinstance(d, list): for i in range(len(d)): dict_expand(d[i], path, outer_dict_name) elif isinstance(d, dict): if len(path) == 1: old_value = d.pop(path[0], None) if old_value is not None: d[outer_dict_name] = d.get(outer_dict_name, {}) d[outer_dict_name][path[0]] = old_value else: sd = d.get(path[0], None) if sd is not None: dict_expand(sd, path[1:], outer_dict_name) def _snake_to_camel(snake, capitalize_first=False): if capitalize_first: return ''.join(x.capitalize() or '_' for x in snake.split('_')) else: return snake.split('_')[0] + ''.join(x.capitalize() or '_' for x in snake.split('_')[1:]) def main(): """Main execution""" AzureRMClusters() if __name__ == '__main__': main()