path: root/ansible_collections/community/aws/plugins/modules/efs.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

DOCUMENTATION = r"""
---
module: efs
version_added: 1.0.0
short_description: create and maintain EFS file systems
description:
    - Module allows create, search and destroy Amazon EFS file systems.
author:
    - "Ryan Sydnor (@ryansydnor)"
    - "Artem Kazakov (@akazakov)"
options:
    encrypt:
        description:
            - If I(encrypt=true) creates an encrypted file system. This can not be modified after the file system is created.
        type: bool
        default: false
    kms_key_id:
        description:
            - The id of the AWS KMS CMK that will be used to protect the encrypted file system. This parameter is only
              required if you want to use a non-default CMK. If this parameter is not specified, the default CMK for
              Amazon EFS is used. The key id can be Key ID, Key ID ARN, Key Alias or Key Alias ARN.
        type: str
    state:
        description:
            - Allows to create, search and destroy Amazon EFS file system.
        default: 'present'
        choices: ['present', 'absent']
        type: str
    name:
        description:
            - Creation Token of Amazon EFS file system. Required for create and update. Either name or ID required for delete.
        type: str
    id:
        description:
            - ID of Amazon EFS. Either name or ID required for delete.
        type: str
    performance_mode:
        description:
            - File system's performance mode to use. Only takes effect during creation.
        default: 'general_purpose'
        choices: ['general_purpose', 'max_io']
        type: str
    tags:
        description:
            - "List of tags of Amazon EFS. Should be defined as dictionary
              In case of 'present' state with list of tags and existing EFS (matched by 'name'), tags of EFS will be replaced with provided data."
        type: dict
    targets:
        description:
            - "List of mounted targets. It should be a list of dictionaries, every dictionary should include next attributes:
               This data may be modified for existing EFS using state 'present' and new list of mount targets."
        type: list
        elements: dict
        default: []
        suboptions:
            subnet_id:
                required: true
                description: The ID of the subnet to add the mount target in.
            ip_address:
                type: str
                description: A valid IPv4 address within the address range of the specified subnet.
            security_groups:
                type: list
                elements: str
                description: List of security group IDs, of the form 'sg-xxxxxxxx'. These must be for the same VPC as subnet specified
    throughput_mode:
        description:
            - The throughput_mode for the file system to be created.
        choices: ['bursting', 'provisioned']
        type: str
    provisioned_throughput_in_mibps:
        description:
            - If the throughput_mode is provisioned, select the amount of throughput to provisioned in Mibps.
        type: float
    wait:
        description:
            - "In case of 'present' state should wait for EFS 'available' life cycle state (of course, if current state not 'deleting' or 'deleted')
               In case of 'absent' state should wait for EFS 'deleted' life cycle state"
        type: bool
        default: false
    wait_timeout:
        description:
            - How long the module should wait (in seconds) for desired state before returning. Zero means wait as long as necessary.
        default: 0
        type: int
    transition_to_ia:
        description:
            - How many days before objects transition to the lower-cost EFS Infrequent Access (IA) storage class.
            - If set to the string C(None), any existing lifecyle policy will be removed, and objects will not transition
              to an IA storage class.
            - If this parameter is absent, any existing lifecycle policy will not be affected.
        choices: ['None', '7', '14', '30', '60', '90']
        type: str
        version_added: 2.1.0

extends_documentation_fragment:
    - amazon.aws.common.modules
    - amazon.aws.region.modules
    - amazon.aws.tags
    - amazon.aws.boto3
"""

EXAMPLES = r"""
- name: EFS provisioning
  community.aws.efs:
    state: present
    name: myTestEFS
    tags:
      Name: myTestNameTag
      purpose: file-storage
    targets:
      - subnet_id: subnet-748c5d03
        security_groups: ["sg-1a2b3c4d"]

- name: Modifying EFS data
  community.aws.efs:
    state: present
    name: myTestEFS
    tags:
      name: myAnotherTestTag
    targets:
      - subnet_id: subnet-7654fdca
        security_groups: ["sg-4c5d6f7a"]

- name: Set a lifecycle policy
  community.aws.efs:
    state: present
    name: myTestEFS
    transition_to_ia: 7
    targets:
      - subnet_id: subnet-7654fdca
        security_groups: ["sg-4c5d6f7a"]

- name: Remove a lifecycle policy
  community.aws.efs:
    state: present
    name: myTestEFS
    transition_to_ia: None
    targets:
      - subnet_id: subnet-7654fdca
        security_groups: ["sg-4c5d6f7a"]

- name: Deleting EFS
  community.aws.efs:
    state: absent
    name: myTestEFS
"""

RETURN = r"""
creation_time:
    description: timestamp of creation date
    returned: always
    type: str
    sample: "2015-11-16 07:30:57-05:00"
creation_token:
    description: EFS creation token
    returned: always
    type: str
    sample: "console-88609e04-9a0e-4a2e-912c-feaa99509961"
file_system_id:
    description: ID of the file system
    returned: always
    type: str
    sample: "fs-xxxxxxxx"
life_cycle_state:
    description: state of the EFS file system
    returned: always
    type: str
    sample: "creating, available, deleting, deleted"
mount_point:
    description: url of file system with leading dot from the time when AWS EFS required to add a region suffix to the address
    returned: always
    type: str
    sample: ".fs-xxxxxxxx.efs.us-west-2.amazonaws.com:/"
filesystem_address:
    description: url of file system valid for use with mount
    returned: always
    type: str
    sample: "fs-xxxxxxxx.efs.us-west-2.amazonaws.com:/"
mount_targets:
    description: list of mount targets
    returned: always
    type: list
    sample:
        [
            {
                "file_system_id": "fs-a7ad440e",
                "ip_address": "172.31.17.173",
                "life_cycle_state": "available",
                "mount_target_id": "fsmt-d8907871",
                "network_interface_id": "eni-6e387e26",
                "owner_id": "123456789012",
                "security_groups": [
                    "sg-a30b22c6"
                ],
                "subnet_id": "subnet-e265c895"
            },
            ...
        ]
name:
    description: name of the file system
    returned: always
    type: str
    sample: "my-efs"
number_of_mount_targets:
    description: the number of targets mounted
    returned: always
    type: int
    sample: 3
owner_id:
    description: AWS account ID of EFS owner
    returned: always
    type: str
    sample: "XXXXXXXXXXXX"
size_in_bytes:
    description: size of the file system in bytes as of a timestamp
    returned: always
    type: dict
    sample:
        {
            "timestamp": "2015-12-21 13:59:59-05:00",
            "value": 12288
        }
performance_mode:
    description: performance mode of the file system
    returned: always
    type: str
    sample: "generalPurpose"
tags:
    description: tags on the efs instance
    returned: always
    type: dict
    sample:
        {
            "name": "my-efs",
            "key": "Value"
        }
"""

from time import sleep
from time import time as timestamp

try:
    import botocore
except ImportError as e:
    pass  # Handled by AnsibleAWSModule

from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict

from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
from ansible_collections.amazon.aws.plugins.module_utils.tagging import ansible_dict_to_boto3_tag_list
from ansible_collections.amazon.aws.plugins.module_utils.tagging import boto3_tag_list_to_ansible_dict
from ansible_collections.amazon.aws.plugins.module_utils.tagging import compare_aws_tags

from ansible_collections.community.aws.plugins.module_utils.modules import AnsibleCommunityAWSModule as AnsibleAWSModule


def _index_by_key(key, items):
    return dict((item[key], item) for item in items)


class EFSConnection(object):
    DEFAULT_WAIT_TIMEOUT_SECONDS = 0

    STATE_CREATING = "creating"
    STATE_AVAILABLE = "available"
    STATE_DELETING = "deleting"
    STATE_DELETED = "deleted"

    def __init__(self, module):
        self.connection = module.client("efs")
        region = module.region

        self.module = module
        self.region = region
        self.wait = module.params.get("wait")
        self.wait_timeout = module.params.get("wait_timeout")

    def get_file_systems(self, **kwargs):
        """
        Returns generator of file systems including all attributes of FS
        """
        items = iterate_all(
            "FileSystems",
            self.connection.describe_file_systems,
            **kwargs,
        )
        for item in items:
            item["Name"] = item["CreationToken"]
            item["CreationTime"] = str(item["CreationTime"])
            """
            In the time when MountPoint was introduced there was a need to add a suffix of network path before one could use it
            AWS updated it and now there is no need to add a suffix. MountPoint is left for back-compatibility purpose
            And new FilesystemAddress variable is introduced for direct use with other modules (e.g. mount)
            AWS documentation is available here:
            https://docs.aws.amazon.com/efs/latest/ug/gs-step-three-connect-to-ec2-instance.html
            """
            item["MountPoint"] = f".{item['FileSystemId']}.efs.{self.region}.amazonaws.com:/"
            item["FilesystemAddress"] = f"{item['FileSystemId']}.efs.{self.region}.amazonaws.com:/"
            if "Timestamp" in item["SizeInBytes"]:
                item["SizeInBytes"]["Timestamp"] = str(item["SizeInBytes"]["Timestamp"])
            if item["LifeCycleState"] == self.STATE_AVAILABLE:
                item["Tags"] = self.get_tags(FileSystemId=item["FileSystemId"])
                item["MountTargets"] = list(self.get_mount_targets(FileSystemId=item["FileSystemId"]))
            else:
                item["Tags"] = {}
                item["MountTargets"] = []
            yield item

    def get_tags(self, **kwargs):
        """
        Returns tag list for selected instance of EFS
        """
        tags = self.connection.describe_tags(**kwargs)["Tags"]
        return tags

    def get_mount_targets(self, **kwargs):
        """
        Returns mount targets for selected instance of EFS
        """
        targets = iterate_all(
            "MountTargets",
            self.connection.describe_mount_targets,
            **kwargs,
        )
        for target in targets:
            if target["LifeCycleState"] == self.STATE_AVAILABLE:
                target["SecurityGroups"] = list(self.get_security_groups(MountTargetId=target["MountTargetId"]))
            else:
                target["SecurityGroups"] = []
            yield target

    def get_security_groups(self, **kwargs):
        """
        Returns security groups for selected instance of EFS
        """
        return iterate_all(
            "SecurityGroups",
            self.connection.describe_mount_target_security_groups,
            **kwargs,
        )

    def get_file_system_id(self, name):
        """
        Returns ID of instance by instance name
        """
        info = first_or_default(
            iterate_all(
                "FileSystems",
                self.connection.describe_file_systems,
                CreationToken=name,
            )
        )
        return info and info["FileSystemId"] or None

    def get_file_system_state(self, name, file_system_id=None):
        """
        Returns state of filesystem by EFS id/name
        """
        info = first_or_default(
            iterate_all(
                "FileSystems",
                self.connection.describe_file_systems,
                CreationToken=name,
                FileSystemId=file_system_id,
            )
        )
        return info and info["LifeCycleState"] or self.STATE_DELETED

    def get_mount_targets_in_state(self, file_system_id, states=None):
        """
        Returns states of mount targets of selected EFS with selected state(s) (optional)
        """
        targets = iterate_all(
            "MountTargets",
            self.connection.describe_mount_targets,
            FileSystemId=file_system_id,
        )

        if states:
            if not isinstance(states, list):
                states = [states]
            targets = filter(lambda target: target["LifeCycleState"] in states, targets)

        return list(targets)

    def get_throughput_mode(self, **kwargs):
        """
        Returns throughput mode for selected EFS instance
        """
        info = first_or_default(
            iterate_all(
                "FileSystems",
                self.connection.describe_file_systems,
                **kwargs,
            )
        )

        return info and info["ThroughputMode"] or None

    def get_provisioned_throughput_in_mibps(self, **kwargs):
        """
        Returns throughput mode for selected EFS instance
        """
        info = first_or_default(
            iterate_all(
                "FileSystems",
                self.connection.describe_file_systems,
                **kwargs,
            )
        )
        return info.get("ProvisionedThroughputInMibps", None)

    def create_file_system(
        self, name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps
    ):
        """
        Creates new filesystem with selected name
        """
        changed = False
        state = self.get_file_system_state(name)
        params = {}
        params["CreationToken"] = name
        params["PerformanceMode"] = performance_mode
        if encrypt:
            params["Encrypted"] = encrypt
        if kms_key_id is not None:
            params["KmsKeyId"] = kms_key_id
        if throughput_mode:
            params["ThroughputMode"] = throughput_mode
        if provisioned_throughput_in_mibps:
            params["ProvisionedThroughputInMibps"] = provisioned_throughput_in_mibps

        if state in [self.STATE_DELETING, self.STATE_DELETED]:
            wait_for(
                lambda: self.get_file_system_state(name),
                self.STATE_DELETED,
            )
            try:
                self.connection.create_file_system(**params)
                changed = True
            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                self.module.fail_json_aws(e, msg="Unable to create file system.")

        # we always wait for the state to be available when creating.
        # if we try to take any actions on the file system before it's available
        # we'll throw errors
        wait_for(
            lambda: self.get_file_system_state(name),
            self.STATE_AVAILABLE,
            self.wait_timeout,
        )

        return changed

    def update_file_system(self, name, throughput_mode, provisioned_throughput_in_mibps):
        """
        Update filesystem with new throughput settings
        """
        changed = False
        state = self.get_file_system_state(name)
        if state in [self.STATE_AVAILABLE, self.STATE_CREATING]:
            fs_id = self.get_file_system_id(name)
            current_mode = self.get_throughput_mode(FileSystemId=fs_id)
            current_throughput = self.get_provisioned_throughput_in_mibps(FileSystemId=fs_id)
            params = dict()
            if throughput_mode and throughput_mode != current_mode:
                params["ThroughputMode"] = throughput_mode
            if provisioned_throughput_in_mibps and provisioned_throughput_in_mibps != current_throughput:
                params["ProvisionedThroughputInMibps"] = provisioned_throughput_in_mibps
            if len(params) > 0:
                wait_for(
                    lambda: self.get_file_system_state(name),
                    self.STATE_AVAILABLE,
                    self.wait_timeout,
                )
                try:
                    self.connection.update_file_system(FileSystemId=fs_id, **params)
                    changed = True
                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                    self.module.fail_json_aws(e, msg="Unable to update file system.")
        return changed

    def update_lifecycle_policy(self, name, transition_to_ia):
        """
        Update filesystem with new lifecycle policy.
        """
        changed = False
        state = self.get_file_system_state(name)
        if state in [self.STATE_AVAILABLE, self.STATE_CREATING]:
            fs_id = self.get_file_system_id(name)
            current_policies = self.connection.describe_lifecycle_configuration(FileSystemId=fs_id)
            if transition_to_ia == "None":
                LifecyclePolicies = []
            else:
                LifecyclePolicies = [{"TransitionToIA": "AFTER_" + transition_to_ia + "_DAYS"}]
            if current_policies.get("LifecyclePolicies") != LifecyclePolicies:
                response = self.connection.put_lifecycle_configuration(
                    FileSystemId=fs_id,
                    LifecyclePolicies=LifecyclePolicies,
                )
                changed = True
        return changed

    def converge_file_system(self, name, tags, purge_tags, targets, throughput_mode, provisioned_throughput_in_mibps):
        """
        Change attributes (mount targets and tags) of filesystem by name
        """
        result = False
        fs_id = self.get_file_system_id(name)

        if tags is not None:
            tags_need_modify, tags_to_delete = compare_aws_tags(
                boto3_tag_list_to_ansible_dict(self.get_tags(FileSystemId=fs_id)), tags, purge_tags
            )

            if tags_to_delete:
                try:
                    self.connection.delete_tags(FileSystemId=fs_id, TagKeys=tags_to_delete)
                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                    self.module.fail_json_aws(e, msg="Unable to delete tags.")

                result = True

            if tags_need_modify:
                try:
                    self.connection.create_tags(
                        FileSystemId=fs_id, Tags=ansible_dict_to_boto3_tag_list(tags_need_modify)
                    )
                except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                    self.module.fail_json_aws(e, msg="Unable to create tags.")

                result = True

        if targets is not None:
            incomplete_states = [self.STATE_CREATING, self.STATE_DELETING]
            wait_for(
                lambda: len(self.get_mount_targets_in_state(fs_id, incomplete_states)),
                0,
            )
            current_targets = _index_by_key("SubnetId", self.get_mount_targets(FileSystemId=fs_id))
            targets = _index_by_key("SubnetId", targets)

            targets_to_create, intersection, targets_to_delete = dict_diff(current_targets, targets, True)

            # To modify mount target it should be deleted and created again
            changed = [
                sid
                for sid in intersection
                if not targets_equal(
                    ["SubnetId", "IpAddress", "NetworkInterfaceId"], current_targets[sid], targets[sid]
                )
            ]
            targets_to_delete = list(targets_to_delete) + changed
            targets_to_create = list(targets_to_create) + changed

            if targets_to_delete:
                for sid in targets_to_delete:
                    self.connection.delete_mount_target(MountTargetId=current_targets[sid]["MountTargetId"])
                wait_for(
                    lambda: len(self.get_mount_targets_in_state(fs_id, incomplete_states)),
                    0,
                )
                result = True

            if targets_to_create:
                for sid in targets_to_create:
                    self.connection.create_mount_target(FileSystemId=fs_id, **targets[sid])
                wait_for(
                    lambda: len(self.get_mount_targets_in_state(fs_id, incomplete_states)),
                    0,
                    self.wait_timeout,
                )
                result = True

            # If no security groups were passed into the module, then do not change it.
            security_groups_to_update = [
                sid
                for sid in intersection
                if "SecurityGroups" in targets[sid]
                and current_targets[sid]["SecurityGroups"] != targets[sid]["SecurityGroups"]
            ]

            if security_groups_to_update:
                for sid in security_groups_to_update:
                    self.connection.modify_mount_target_security_groups(
                        MountTargetId=current_targets[sid]["MountTargetId"],
                        SecurityGroups=targets[sid].get("SecurityGroups", None),
                    )
                result = True

        return result

    def delete_file_system(self, name, file_system_id=None):
        """
        Removes EFS instance by id/name
        """
        result = False
        state = self.get_file_system_state(name, file_system_id)
        if state in [self.STATE_CREATING, self.STATE_AVAILABLE]:
            wait_for(
                lambda: self.get_file_system_state(name),
                self.STATE_AVAILABLE,
            )
            if not file_system_id:
                file_system_id = self.get_file_system_id(name)
            self.delete_mount_targets(file_system_id)
            self.connection.delete_file_system(FileSystemId=file_system_id)
            result = True

        if self.wait:
            wait_for(
                lambda: self.get_file_system_state(name),
                self.STATE_DELETED,
                self.wait_timeout,
            )

        return result

    def delete_mount_targets(self, file_system_id):
        """
        Removes mount targets by EFS id
        """
        wait_for(
            lambda: len(self.get_mount_targets_in_state(file_system_id, self.STATE_CREATING)),
            0,
        )

        targets = self.get_mount_targets_in_state(file_system_id, self.STATE_AVAILABLE)
        for target in targets:
            self.connection.delete_mount_target(MountTargetId=target["MountTargetId"])

        wait_for(
            lambda: len(self.get_mount_targets_in_state(file_system_id, self.STATE_DELETING)),
            0,
        )

        return len(targets) > 0


def iterate_all(attr, map_method, **kwargs):
    """
    Method creates iterator from result set
    """
    args = dict((key, value) for (key, value) in kwargs.items() if value is not None)
    wait = 1
    while True:
        try:
            data = map_method(**args)
            yield from data[attr]
            if "NextMarker" in data:
                args["Marker"] = data["Nextmarker"]
                continue
            break
        except is_boto3_error_code("ThrottlingException"):
            if wait < 600:
                sleep(wait)
                wait = wait * 2
                continue
            else:
                raise


def targets_equal(keys, a, b):
    """
    Method compare two mount targets by specified attributes
    """
    for key in keys:
        if key in b and a[key] != b[key]:
            return False

    return True


def dict_diff(dict1, dict2, by_key=False):
    """
    Helper method to calculate difference of two dictionaries
    """
    keys1 = set(dict1.keys() if by_key else dict1.items())
    keys2 = set(dict2.keys() if by_key else dict2.items())

    intersection = keys1 & keys2

    return keys2 ^ intersection, intersection, keys1 ^ intersection


def first_or_default(items, default=None):
    """
    Helper method to fetch first element of list (if exists)
    """
    for item in items:
        return item
    return default


def wait_for(callback, value, timeout=EFSConnection.DEFAULT_WAIT_TIMEOUT_SECONDS):
    """
    Helper method to wait for desired value returned by callback method
    """
    wait_start = timestamp()
    while True:
        if callback() != value:
            if timeout != 0 and (timestamp() - wait_start > timeout):
                raise RuntimeError("Wait timeout exceeded (" + str(timeout) + " sec)")
            else:
                sleep(5)
            continue
        break


def main():
    """
    Module action handler
    """
    argument_spec = dict(
        encrypt=dict(required=False, type="bool", default=False),
        state=dict(required=False, type="str", choices=["present", "absent"], default="present"),
        kms_key_id=dict(required=False, type="str", default=None),
        purge_tags=dict(default=True, type="bool"),
        id=dict(required=False, type="str", default=None),
        name=dict(required=False, type="str", default=None),
        tags=dict(required=False, type="dict", aliases=["resource_tags"]),
        targets=dict(required=False, type="list", default=[], elements="dict"),
        performance_mode=dict(
            required=False, type="str", choices=["general_purpose", "max_io"], default="general_purpose"
        ),
        transition_to_ia=dict(required=False, type="str", choices=["None", "7", "14", "30", "60", "90"], default=None),
        throughput_mode=dict(required=False, type="str", choices=["bursting", "provisioned"], default=None),
        provisioned_throughput_in_mibps=dict(required=False, type="float"),
        wait=dict(required=False, type="bool", default=False),
        wait_timeout=dict(required=False, type="int", default=0),
    )

    module = AnsibleAWSModule(argument_spec=argument_spec)

    connection = EFSConnection(module)

    name = module.params.get("name")
    fs_id = module.params.get("id")
    tags = module.params.get("tags")
    target_translations = {
        "ip_address": "IpAddress",
        "security_groups": "SecurityGroups",
        "subnet_id": "SubnetId",
    }
    targets = [
        dict((target_translations[key], value) for (key, value) in x.items()) for x in module.params.get("targets")
    ]
    performance_mode_translations = {
        "general_purpose": "generalPurpose",
        "max_io": "maxIO",
    }
    encrypt = module.params.get("encrypt")
    kms_key_id = module.params.get("kms_key_id")
    performance_mode = performance_mode_translations[module.params.get("performance_mode")]
    purge_tags = module.params.get("purge_tags")
    transition_to_ia = module.params.get("transition_to_ia")
    throughput_mode = module.params.get("throughput_mode")
    provisioned_throughput_in_mibps = module.params.get("provisioned_throughput_in_mibps")
    state = str(module.params.get("state")).lower()
    changed = False

    if state == "present":
        if not name:
            module.fail_json(msg="Name parameter is required for create")

        changed = connection.create_file_system(
            name, performance_mode, encrypt, kms_key_id, throughput_mode, provisioned_throughput_in_mibps
        )
        changed = connection.update_file_system(name, throughput_mode, provisioned_throughput_in_mibps) or changed
        changed = (
            connection.converge_file_system(
                name=name,
                tags=tags,
                purge_tags=purge_tags,
                targets=targets,
                throughput_mode=throughput_mode,
                provisioned_throughput_in_mibps=provisioned_throughput_in_mibps,
            )
            or changed
        )
        if transition_to_ia:
            changed |= connection.update_lifecycle_policy(name, transition_to_ia)
        result = first_or_default(connection.get_file_systems(CreationToken=name))

    elif state == "absent":
        if not name and not fs_id:
            module.fail_json(msg="Either name or id parameter is required for delete")

        changed = connection.delete_file_system(name, fs_id)
        result = None
    if result:
        result = camel_dict_to_snake_dict(result)
    module.exit_json(changed=changed, efs=result)


if __name__ == "__main__":
    main()